Ticket #1835: 1835-v4.patch

programs/mythbackend/mainserver.cpp

@@ -58 +58 @@
 /** Number of threads in process request thread pool at startup. */
 #define PRT_STARTUP_THREAD_COUNT 5
 
+QMutex MainServer::truncate_and_close_lock;
+
 class ProcessRequestThread : public QThread
 {
   public:
@@ -1340 +1342 @@
         tvchain->DeleteProgram(pginfo);
 
     int err;
+    int fd = -1;
     QString filename = ds->filename;
     bool followLinks = gContext->GetNumSetting("DeletesFollowLinks", 0);
 
-    VERBOSE(VB_FILE, QString("About to unlink/delete file: %1").arg(filename));
-    if (followLinks)
+    VERBOSE(VB_FILE, QString("About to unlink/delete file: '%1'")
+            .arg(filename));
+
+    QFileInfo finfo(filename);
+    if (finfo.isSymLink())
     {
-        QFileInfo finfo(filename);
-        if (finfo.isSymLink() && (err = unlink(finfo.readLink().local8Bit())))
+        if (followLinks)
+            fd = OpenAndUnlink(finfo.readLink());
+
+        err = unlink(filename.local8Bit());
+
+        if (err)
         {
-            VERBOSE(VB_IMPORTANT, QString("Error deleting '%1' @ '%2', %3")
-                    .arg(filename).arg(finfo.readLink().local8Bit())
-                    .arg(strerror(errno)));
+            VERBOSE(VB_IMPORTANT,
+                    QString("Error deleting '%1'").arg(filename) + ENO);
         }
     }
-    if ((err = unlink(filename.local8Bit())))
-        VERBOSE(VB_IMPORTANT, QString("Error deleting '%1', %2")
-                .arg(filename).arg(strerror(errno)));
+    else
+    {
+        fd = OpenAndUnlink(filename);
+    }
     
     sleep(2);
 
@@ -1376 +1386 @@
         gContext->dispatch(me);
 
         deletelock.unlock();
+        if (fd != -1)
+            close(fd);
         return;
     }
 
@@ -1478 +1490 @@
     delete pginfo;
 
     deletelock.unlock();
+
+    if (fd != -1)
+    {
+        m_expirer->TruncatePending();
+        TruncateAndClose(m_expirer, fd, ds->filename);
+        m_expirer->TruncateFinished();
+    }
 }
 
+/** \fn MainServer::OpenAndUnlink(const QString&)
+ *  \brief Opens a file, unlinks it and returns the file descriptor.
+ */
+int MainServer::OpenAndUnlink(const QString &filename)
+{
+    QString msg = QString("Error deleting '%1'").arg(filename);
+    int fd = open(filename.local8Bit(), O_WRONLY);
+
+    if (fd == -1)
+    {
+        VERBOSE(VB_IMPORTANT, msg + ENO);
+        return -1;
+    }
+    
+    if (unlink(filename.local8Bit()))
+    {
+        VERBOSE(VB_IMPORTANT, msg + ENO);
+        close(fd);
+        return -1;
+    }
+
+    return fd;
+}
+
+/** \fn MainServer::TruncateAndClose(const AutoExpire*,int,const QString&)
+ *  \brief Repeatedly truncate an open file in small increments.
+ *
+ *   When the file is small enough this closes the file and returns.
+ *
+ *   NOTE: This aquires a lock so that only one instance of TruncateAndClose()
+ *         is running at a time.
+ */
+bool MainServer::TruncateAndClose(const AutoExpire *expirer,
+                                  int fd, const QString &filename)
+{
+    QMutexLocker locker(&truncate_and_close_lock);
+
+    // Time between truncation steps in milliseconds
+    const size_t sleep_time = 500;
+
+    // Compute the truncate increment such that we delete
+    // 20% faster than the maximum recording rate.
+    size_t increment;
+    increment = (expirer->GetMinTruncateRate() * sleep_time + 999) / 1000;
+
+    VERBOSE(VB_FILE,
+            QString("Truncating '%1' by %2 MB every %3 milliseconds")
+            .arg(filename)
+            .arg(increment / (1024.0 * 1024.0), 0, 'f', 2)
+            .arg(sleep_time));
+
+    const QString err_msg = QString("Error truncating '%1'").arg(filename);
+
+    size_t fsize = 0;
+    while (true)
+    {
+        // Get the on disk file size and disk block size.
+        struct stat buf;
+        fstat(fd, &buf);
+        fsize = buf.st_blksize * buf.st_blocks;
+
+        if (fsize <= increment)
+            break;
+
+        // Round truncate increment up to a blocksize, w/min of 1 block.
+        increment = ((increment / buf.st_blksize) + 1) * buf.st_blksize;
+
+        int err = ftruncate(fd, fsize - increment);
+        if (err)
+        {
+            VERBOSE(VB_IMPORTANT, err_msg + ENO);
+            return 0 == close(fd);
+        }
+
+        usleep(sleep_time * 1000);
+    }
+
+    bool ok = (0 == close(fd));
+
+    usleep((sleep_time * fsize * 1000) / increment);
+
+    VERBOSE(VB_FILE, QString("Finished truncating '%1'").arg(filename));
+
+    return ok;
+}
+
 void MainServer::HandleCheckRecordingActive(QStringList &slist, 
                                             PlaybackSock *pbs)
 {

programs/mythbackend/mainserver.h

@@ -149 +149 @@
     void AddToChains(LiveTVChain *chain);
     void DeleteChain(LiveTVChain *chain);
 
+    static int  OpenAndUnlink(const QString &filename);
+    static bool TruncateAndClose(const AutoExpire *expirer,
+                                 int fd, const QString &filename);
+
     QPtrList<LiveTVChain> liveTVChains;
     QMutex liveTVChainsLock;
 
@@ -188 +192 @@
     QMutex deferredDeleteLock;
     QTimer *deferredDeleteTimer;
     QValueList<DeferredDeleteStruct> deferredDeleteList;
+
+    static QMutex truncate_and_close_lock;
 };
 
 #endif

programs/mythbackend/autoexpire.cpp

@@ -53 +53 @@
  */
 AutoExpire::AutoExpire(bool runthread, bool master)
     : record_file_prefix("/"), desired_space(3*1024*1024),
-      desired_freq(10), expire_thread_running(runthread),
-      is_master_backend(master), update_pending(false)    
+      desired_freq(10), max_record_rate(5*1024*1024),
+      expire_thread_running(runthread), is_master_backend(master),
+      truncates_pending(0), update_pending(false)
 {
     if (runthread)
     {
@@ -179 +180 @@
     instance_lock.lock();
     desired_space      = expireMinKB;
     desired_freq       = expireFreq;
+    max_record_rate    = (totalKBperMin * 1024)/60;
     record_file_prefix = recordFilePrefix;
     instance_lock.unlock();
     DBG_CALC_PARAM("CalcParams() -- end");
@@ -227 +229 @@
  */
 void AutoExpire::RunExpirer(void)
 {
-    QTime curTime;
     QTime timer;
+    QDateTime curTime;
+    QDateTime next_expire = QDateTime::currentDateTime().addSecs(60);
 
     // wait a little for main server to come up and things to settle down
     sleep(20);
@@ -243 +246 @@
 
         UpdateDontExpireSet();
 
-        curTime = QTime::currentTime();
+        curTime = QDateTime::currentDateTime();
 
         // Expire Short LiveTV files for this backend every 2 minutes
-        if ((curTime.minute() % 2) == 0)
+        if ((curTime.time().minute() % 2) == 0)
             ExpireLiveTV(emShortLiveTVPrograms);
 
         // Expire normal recordings depending on frequency calculated
-        if ((curTime.minute() % desired_freq) == 0)
+        if (curTime >= next_expire)
         {
+            // Wait for all pending truncates to finish
+            WaitForPendingTruncates();
+
+            next_expire =
+                QDateTime::currentDateTime().addSecs(desired_freq * 60);
+
             ExpireLiveTV(emNormalLiveTVPrograms);
 
             if (is_master_backend)
@@ -281 +290 @@
     }
 }
 
+void AutoExpire::TruncatePending(void)
+{
+    QMutexLocker locker(&truncate_monitor_lock);
+
+    truncates_pending++;
+
+    VERBOSE(VB_FILE, LOC<<truncates_pending<<" file truncates are pending");
+}
+
+void AutoExpire::TruncateFinished(void)
+{
+    QMutexLocker locker(&truncate_monitor_lock);
+
+    truncates_pending--;
+
+    if (truncates_pending <= 0)
+    {
+        VERBOSE(VB_FILE, LOC + "All file truncates have finished");
+        truncate_monitor_condition.wakeAll();
+    }
+}
+
+void AutoExpire::WaitForPendingTruncates(void)
+{
+    QMutexLocker locker(&truncate_monitor_lock);
+    while (truncates_pending > 0)
+    {
+        VERBOSE(VB_FILE, LOC + "Waiting for pending file truncates");
+        truncate_monitor_condition.wait(&truncate_monitor_lock);
+    }
+}
+
 /** \fn AutoExpire::ExpireLiveTV(int type)
  *  \brief This expires LiveTV programs.
  */

programs/mythbackend/autoexpire.h

@@ -9 +9 @@
 
 #include <qmap.h> 
 #include <qmutex.h>
+#include <qwaitcondition.h>
 #include <qobject.h>
 
 using namespace std;
@@ -35 +36 @@
     void CalcParams(vector<EncoderLink*>);
     void FillExpireList();
     void PrintExpireList();
+    void TruncatePending(void);
+    void TruncateFinished(void);
+
+    size_t GetMaxRecordRate(void) const
+        { return max_record_rate; }
+
+    size_t GetMinTruncateRate(void) const
+        { return ((max_record_rate * 6) / 5) + 1; }
+
     void GetAllExpiring(QStringList &strList);
 
     static void Update(QMap<int, EncoderLink*>*, bool immediately);
@@ -52 +62 @@
                             bool deleteAll = false);
     void ClearExpireList(void);
     void Sleep(int sleepTime);
+    void WaitForPendingTruncates(void);
 
     void UpdateDontExpireSet(void);
     bool IsInDontExpireSet(QString chanid, QDateTime starttime);
@@ -65 +76 @@
     QString       record_file_prefix;
     size_t        desired_space;
     uint          desired_freq;
+    size_t        max_record_rate; // bytes/sec
     bool          expire_thread_running;
     bool          is_master_backend;
 
+    // Pending truncates monitor
+    mutable QMutex truncate_monitor_lock;
+    QWaitCondition truncate_monitor_condition;
+    int            truncates_pending;
+
     // update info
     bool          update_pending;
     pthread_t     update_thread;