Ticket #1835: gradual-delete3.patch

programs/mythbackend/mainserver.cpp

@@ -1337 +1337 @@
         tvchain->DeleteProgram(pginfo);
 
     int err;
+    int fd = -1;
     QString filename = ds->filename;
     bool followLinks = gContext->GetNumSetting("DeletesFollowLinks", 0);
 
-    VERBOSE(VB_FILE, QString("About to unlink/delete file: %1").arg(filename));
-    if (followLinks)
+    VERBOSE(VB_FILE, QString("About to unlink/delete file: '%1'").arg(filename));
+    QFileInfo finfo(filename);
+    if (finfo.isSymLink())
     {
-        QFileInfo finfo(filename);
-        if (finfo.isSymLink() && (err = unlink(finfo.readLink().local8Bit())))
-        {
-            VERBOSE(VB_IMPORTANT, QString("Error deleting '%1' @ '%2', %3")
-                    .arg(filename).arg(finfo.readLink().local8Bit())
-                    .arg(strerror(errno)));
-        }
+        if (followLinks)
+            fd = OpenAndUnlink(finfo.readLink());
+
+        if ((err = unlink(filename.local8Bit())))
+            VERBOSE(VB_IMPORTANT, QString("Error deleting '%1', %2")
+                    .arg(filename).arg(strerror(errno)));
     }
-    if ((err = unlink(filename.local8Bit())))
-        VERBOSE(VB_IMPORTANT, QString("Error deleting '%1', %2")
-                .arg(filename).arg(strerror(errno)));
+    else
+        fd = OpenAndUnlink(filename);
     
     sleep(2);
 
@@ -1373 +1373 @@
         gContext->dispatch(me);
 
         deletelock.unlock();
+        if (fd != -1) close(fd);
         return;
     }
 
@@ -1475 +1476 @@
     delete pginfo;
 
     deletelock.unlock();
+
+    if (fd != -1)
+    {
+        m_expirer->TruncatePending();
+        TruncateAndClose(fd, ds->filename);
+        m_expirer->TruncateFinished();
+    }
 }
 
+// Opens a file, unlinks it and returns the file descriptor
+int MainServer::OpenAndUnlink(QString filename)
+{
+    int fd = open(filename.local8Bit(), O_WRONLY);
+    if (fd == -1)
+    {
+        VERBOSE(VB_IMPORTANT, QString("Error deleting '%1', %2")
+                .arg(filename).arg(strerror(errno)));
+        return -1;
+    }
+    
+    if (unlink(filename.local8Bit()))
+    {
+        VERBOSE(VB_IMPORTANT, QString("Error deleting '%1', %2")
+                .arg(filename).arg(strerror(errno)));
+        close(fd);
+        return -1;
+    }
+
+    return fd;
+}
+
+// Repeatedly truncate an open file in small increments.
+// When small enough, close the file and return.
+int MainServer::TruncateAndClose(int fd, QString filename)
+{
+    QMutexLocker locker(&truncatelock);
+    const int sleep_time = 2; // time between truncation steps in seconds
+    int err;
+    size_t increment;
+    struct stat buf;
+
+    // Compute the truncate increment such that we delete 20% faster than the
+    // maximum recording rate.
+    increment = (m_expirer->MaxRecordRate() * sleep_time * 6) / 5;
+
+    VERBOSE(VB_FILE,
+            QString("Truncating '%1' by %2 MB every %3 seconds")
+            .arg(filename)
+            .arg(increment / (1024.0 * 1024.0), 0, 'f', 2)
+            .arg(sleep_time));
+
+    while (1)
+    {
+        // Get the file size
+        fstat(fd, &buf);
+        if (buf.st_size <= increment)
+            break;
+
+        if ((err = ftruncate(fd, buf.st_size - increment)))
+        {
+            VERBOSE(VB_IMPORTANT, QString("Error truncating '%1', %2")
+                    .arg(filename).arg(strerror(errno)));
+            return close(fd);
+        }
+            
+        sleep(sleep_time);
+    }
+
+    VERBOSE(VB_FILE, QString("Finished truncating '%1'").arg(filename));
+    return close(fd);
+}
+
 void MainServer::HandleCheckRecordingActive(QStringList &slist, 
                                             PlaybackSock *pbs)
 {

programs/mythbackend/mainserver.h

@@ -142 +142 @@
 
     static void *SpawnDeleteThread(void *param);
     void DoDeleteThread(DeleteStruct *ds);
+    int OpenAndUnlink(QString filename);
+    int TruncateAndClose(int fd, QString filename);
 
     LiveTVChain *GetExistingChain(QString id);
     LiveTVChain *GetExistingChain(QSocket *sock);
@@ -169 +171 @@
     bool ismaster;
 
     QMutex deletelock;
+    QMutex truncatelock;
     QMutex threadPoolLock;
     vector<ProcessRequestThread *> threadPool;
 

programs/mythbackend/autoexpire.cpp

@@ -53 +53 @@
  */
 AutoExpire::AutoExpire(bool runthread, bool master)
     : record_file_prefix("/"), desired_space(3*1024*1024),
-      desired_freq(10), expire_thread_running(runthread),
-      is_master_backend(master), update_pending(false)    
+      desired_freq(10), max_record_rate(5*1024*1024),
+      expire_thread_running(runthread), is_master_backend(master),
+      truncates_pending(0), update_pending(false)
 {
     if (runthread)
     {
@@ -179 +180 @@
     instance_lock.lock();
     desired_space      = expireMinKB;
     desired_freq       = expireFreq;
+    max_record_rate    = (totalKBperMin * 1024)/60;
     record_file_prefix = recordFilePrefix;
     instance_lock.unlock();
     DBG_CALC_PARAM("CalcParams() -- end");
@@ -227 +229 @@
  */
 void AutoExpire::RunExpirer(void)
 {
-    QTime curTime;
     QTime timer;
+    QDateTime curTime;
+    QDateTime next_expire = QDateTime::currentDateTime().addSecs(60);
 
     // wait a little for main server to come up and things to settle down
     sleep(20);
@@ -243 +246 @@
 
         UpdateDontExpireSet();
 
-        curTime = QTime::currentTime();
+        curTime = QDateTime::currentDateTime();
 
         // Expire Short LiveTV files for this backend every 2 minutes
-        if ((curTime.minute() % 2) == 0)
+        if ((curTime.time().minute() % 2) == 0)
             ExpireLiveTV(emShortLiveTVPrograms);
 
         // Expire normal recordings depending on frequency calculated
-        if ((curTime.minute() % desired_freq) == 0)
+        if (curTime >= next_expire)
         {
+            // Wait for all pending truncates to finish
+            WaitForPendingTruncates();
+
+            next_expire =
+                QDateTime::currentDateTime().addSecs(desired_freq * 60);
+
             ExpireLiveTV(emNormalLiveTVPrograms);
 
             if (is_master_backend)
@@ -281 +290 @@
     }
 }
 
+void AutoExpire::TruncatePending(void)
+{
+    QMutexLocker locker(&truncate_monitor_lock);
+    if (truncates_pending++ == 0)
+        VERBOSE(VB_FILE, LOC + "File truncates are pending");
+}
+
+void AutoExpire::TruncateFinished(void)
+{
+    QMutexLocker locker(&truncate_monitor_lock);
+    if (--truncates_pending == 0)
+    {
+        VERBOSE(VB_FILE, LOC + "All file truncates are finished");
+        truncate_monitor_condition.wakeOne();
+    }
+}
+
+void AutoExpire::WaitForPendingTruncates(void)
+{
+    QMutexLocker locker(&truncate_monitor_lock);
+    if (truncates_pending > 0)
+    {
+        VERBOSE(VB_FILE, LOC + "Waiting for pending file truncates");
+        truncate_monitor_condition.wait(&truncate_monitor_lock);
+    }
+}
+
 /** \fn AutoExpire::ExpireLiveTV(int type)
  *  \brief This expires LiveTV programs.
  */

programs/mythbackend/autoexpire.h

@@ -9 +9 @@
 
 #include <qmap.h> 
 #include <qmutex.h>
+#include <qwaitcondition.h>
 #include <qobject.h>
 
 using namespace std;
@@ -33 +34 @@
    ~AutoExpire();
 
     void CalcParams(vector<EncoderLink*>);
+    size_t MaxRecordRate(void) { return max_record_rate; }
     void FillExpireList();
     void PrintExpireList();
     void GetAllExpiring(QStringList &strList);
+    void TruncatePending(void);
+    void TruncateFinished(void);
 
     static void Update(QMap<int, EncoderLink*>*, bool immediately);
   protected:
@@ -52 +56 @@
                             bool deleteAll = false);
     void ClearExpireList(void);
     void Sleep(int sleepTime);
+    void WaitForPendingTruncates(void);
 
     void UpdateDontExpireSet(void);
     bool IsInDontExpireSet(QString chanid, QDateTime starttime);
@@ -65 +70 @@
     QString       record_file_prefix;
     size_t        desired_space;
     uint          desired_freq;
+    size_t        max_record_rate; // bytes/sec
     bool          expire_thread_running;
     bool          is_master_backend;
 
+    // Pending truncates monitor
+    QMutex         truncate_monitor_lock;
+    QWaitCondition truncate_monitor_condition;
+    int            truncates_pending;
+
     // update info
     bool          update_pending;
     pthread_t     update_thread;