Ticket #1835: 1835-v5.patch

programs/mythbackend/mainserver.cpp

@@ -99 +99 @@
 
 };
 
+QMutex MainServer::truncate_and_close_lock;
+
 class ProcessRequestThread : public QThread
 {
   public:
@@ -1383 +1385 @@
     if (tvchain)
         tvchain->DeleteProgram(pginfo);
 
-    int err;
     bool followLinks = gContext->GetNumSetting("DeletesFollowLinks", 0);
 
     /* Delete recording. */
-    err = deleteFile(ds->filename, followLinks, false);
+    int fd = DeleteFile(ds->filename, followLinks);
     
     sleep(2);
 
-    if (checkFile.exists())
+    if ((fd < 0) && checkFile.exists())
     {
         VERBOSE(VB_IMPORTANT,
             QString("Error deleting file: %1. Keeping metadata in database.")
@@ -1411 +1412 @@
     }
 
     /* Delete preview thumbnail. */
-    err = deleteFile(ds->filename + ".png", followLinks, true);
+    deleteFile(ds->filename + ".png", followLinks, true);
 
     MSqlQuery query(MSqlQuery::InitCon());
     query.prepare("DELETE FROM recorded WHERE chanid = :CHANID AND "
@@ -1495 +1496 @@
     delete pginfo;
 
     deletelock.unlock();
+
+    if (fd != -1)
+    {
+        m_expirer->TruncatePending();
+        TruncateAndClose(m_expirer, fd, ds->filename);
+        m_expirer->TruncateFinished();
+    }
 }
 
+/* Return fd for success, negative number for error. */
+int MainServer::DeleteFile(const QString &filename, bool followLinks)
+{
+    QFileInfo finfo(filename);
+    int fd = -1, err = 0;
+
+    VERBOSE(VB_FILE, QString("About to unlink/delete file: '%1'")
+            .arg(filename));
+
+    QString errmsg = QString("Delete Error '%1'").arg(filename.local8Bit());
+    if (finfo.isSymLink())
+        errmsg += QString(" -> '%2'").arg(finfo.readLink().local8Bit());
+
+    if (followLinks && finfo.isSymLink())
+    {
+        if (followLinks)
+            fd = OpenAndUnlink(finfo.readLink());
+        if (fd >= 0)
+            err = unlink(filename.local8Bit());
+    }
+    else if (!finfo.isSymLink())
+    {
+        fd = OpenAndUnlink(filename);
+    }
+    else // just delete symlinks immediately
+    {
+        err = unlink(filename.local8Bit());
+        if (err == 0)
+            return -1; // no error
+    }
+
+    if (fd < 0)
+        VERBOSE(VB_IMPORTANT, errmsg + ENO);
+
+    return fd;
+}
+
+/** \fn MainServer::OpenAndUnlink(const QString&)
+ *  \brief Opens a file, unlinks it and returns the file descriptor.
+ */
+int MainServer::OpenAndUnlink(const QString &filename)
+{
+    QString msg = QString("Error deleting '%1'").arg(filename.local8Bit());
+    int fd = open(filename.local8Bit(), O_WRONLY);
+
+    if (fd == -1)
+    {
+        VERBOSE(VB_IMPORTANT, msg + " could not open " + ENO);
+        return -1;
+    }
+    
+    if (unlink(filename.local8Bit()))
+    {
+        VERBOSE(VB_IMPORTANT, msg + " could not unlink " + ENO);
+        close(fd);
+        return -1;
+    }
+
+    return fd;
+}
+
+/** \fn MainServer::TruncateAndClose(const AutoExpire*,int,const QString&)
+ *  \brief Repeatedly truncate an open file in small increments.
+ *
+ *   When the file is small enough this closes the file and returns.
+ *
+ *   NOTE: This aquires a lock so that only one instance of TruncateAndClose()
+ *         is running at a time.
+ */
+bool MainServer::TruncateAndClose(const AutoExpire *expirer,
+                                  int fd, const QString &filename)
+{
+    QMutexLocker locker(&truncate_and_close_lock);
+
+    // Time between truncation steps in milliseconds
+    const size_t sleep_time = 500;
+    const size_t min_tps    = 8 * 1024 * 1024;
+    const size_t min_trunc  = (size_t) (min_tps * (sleep_time * 0.001f));
+
+    // Compute the truncate increment such that we delete
+    // 20% faster than the maximum recording rate.
+    size_t increment;
+    increment = (expirer->GetMinTruncateRate() * sleep_time + 999) / 1000;
+    increment = max(increment, min_trunc);
+
+    VERBOSE(VB_FILE,
+            QString("Truncating '%1' by %2 MB every %3 milliseconds")
+            .arg(filename)
+            .arg(increment / (1024.0 * 1024.0), 0, 'f', 2)
+            .arg(sleep_time));
+
+    const QString err_msg = QString("Error truncating '%1'").arg(filename);
+
+    // Get the on disk file size and disk block size.
+    struct stat buf;
+    fstat(fd, &buf);
+    size_t fsize = buf.st_blksize * buf.st_blocks;
+
+    // Round truncate increment up to a blocksize, w/min of 1 block.
+    increment = ((increment / buf.st_blksize) + 1) * buf.st_blksize;
+
+    while (fsize > increment)
+    {
+        fsize -= increment;
+
+        //VERBOSE(VB_FILE, QString("Truncating '%1' to %2 MB")
+        //        .arg(filename).arg(fsize / (1024.0 * 1024.0), 0, 'f', 2));
+
+        int err = ftruncate(fd, fsize);
+        if (err)
+        {
+            VERBOSE(VB_IMPORTANT, err_msg + ENO);
+            return 0 == close(fd);
+        }
+
+        usleep(sleep_time * 1000);
+    }
+
+    bool ok = (0 == close(fd));
+
+    usleep((sleep_time * fsize * 1000) / increment);
+
+    VERBOSE(VB_FILE, QString("Finished truncating '%1'").arg(filename));
+
+    return ok;
+}
+
 void MainServer::HandleCheckRecordingActive(QStringList &slist, 
                                             PlaybackSock *pbs)
 {

programs/mythbackend/mainserver.h

@@ -148 +148 @@
     void AddToChains(LiveTVChain *chain);
     void DeleteChain(LiveTVChain *chain);
 
+    static int  DeleteFile(const QString &filename, bool followLinks);
+    static int  OpenAndUnlink(const QString &filename);
+    static bool TruncateAndClose(const AutoExpire *expirer,
+                                 int fd, const QString &filename);
+
     QPtrList<LiveTVChain> liveTVChains;
     QMutex liveTVChainsLock;
 
@@ -187 +192 @@
     QValueList<DeferredDeleteStruct> deferredDeleteList;
 
     QTimer *autoexpireUpdateTimer;
+
+    static QMutex truncate_and_close_lock;
 };
 
 #endif

programs/mythbackend/autoexpire.cpp

@@ -53 +53 @@
  */
 AutoExpire::AutoExpire(bool runthread, bool master)
     : record_file_prefix("/"), desired_space(3*1024*1024),
-      desired_freq(10), expire_thread_running(runthread),
-      is_master_backend(master), update_pending(false)    
+      desired_freq(10), max_record_rate(5*1024*1024),
+      expire_thread_running(runthread), is_master_backend(master),
+      truncates_pending(0), update_pending(false)
 {
     if (runthread)
     {
@@ -179 +180 @@
     instance_lock.lock();
     desired_space      = expireMinKB;
     desired_freq       = expireFreq;
+    max_record_rate    = (totalKBperMin * 1024)/60;
     record_file_prefix = recordFilePrefix;
     instance_lock.unlock();
     DBG_CALC_PARAM("CalcParams() -- end");
@@ -227 +229 @@
  */
 void AutoExpire::RunExpirer(void)
 {
-    QTime curTime;
     QTime timer;
+    QDateTime curTime;
+    QDateTime next_expire = QDateTime::currentDateTime().addSecs(60);
 
     // wait a little for main server to come up and things to settle down
     sleep(20);
@@ -243 +246 @@
 
         UpdateDontExpireSet();
 
-        curTime = QTime::currentTime();
+        curTime = QDateTime::currentDateTime();
 
         // Expire Short LiveTV files for this backend every 2 minutes
-        if ((curTime.minute() % 2) == 0)
+        if ((curTime.time().minute() % 2) == 0)
             ExpireLiveTV(emShortLiveTVPrograms);
 
         // Expire normal recordings depending on frequency calculated
-        if ((curTime.minute() % desired_freq) == 0)
+        if (curTime >= next_expire)
         {
+            // Wait for all pending truncates to finish
+            WaitForPendingTruncates();
+
+            next_expire =
+                QDateTime::currentDateTime().addSecs(desired_freq * 60);
+
             ExpireLiveTV(emNormalLiveTVPrograms);
 
             if (is_master_backend)
@@ -281 +290 @@
     }
 }
 
+void AutoExpire::TruncatePending(void)
+{
+    QMutexLocker locker(&truncate_monitor_lock);
+
+    truncates_pending++;
+
+    VERBOSE(VB_FILE, LOC<<truncates_pending<<" file truncates are pending");
+}
+
+void AutoExpire::TruncateFinished(void)
+{
+    QMutexLocker locker(&truncate_monitor_lock);
+
+    truncates_pending--;
+
+    if (truncates_pending <= 0)
+    {
+        VERBOSE(VB_FILE, LOC + "All file truncates have finished");
+        truncate_monitor_condition.wakeAll();
+    }
+}
+
+void AutoExpire::WaitForPendingTruncates(void)
+{
+    QMutexLocker locker(&truncate_monitor_lock);
+    while (truncates_pending > 0)
+    {
+        VERBOSE(VB_FILE, LOC + "Waiting for pending file truncates");
+        truncate_monitor_condition.wait(&truncate_monitor_lock);
+    }
+}
+
 /** \fn AutoExpire::ExpireLiveTV(int type)
  *  \brief This expires LiveTV programs.
  */

programs/mythbackend/autoexpire.h

@@ -9 +9 @@
 
 #include <qmap.h> 
 #include <qmutex.h>
+#include <qwaitcondition.h>
 #include <qobject.h>
 
 using namespace std;
@@ -35 +36 @@
     void CalcParams(vector<EncoderLink*>);
     void FillExpireList();
     void PrintExpireList();
+    void TruncatePending(void);
+    void TruncateFinished(void);
+
+    size_t GetMaxRecordRate(void) const
+        { return max_record_rate; }
+
+    size_t GetMinTruncateRate(void) const
+        { return ((max_record_rate * 6) / 5) + 1; }
+
     void GetAllExpiring(QStringList &strList);
     void GetAllExpiring(pginfolist_t &list);
 
@@ -53 +63 @@
                             bool deleteAll = false);
     void ClearExpireList(void);
     void Sleep(int sleepTime);
+    void WaitForPendingTruncates(void);
 
     void UpdateDontExpireSet(void);
     bool IsInDontExpireSet(QString chanid, QDateTime starttime);
@@ -66 +77 @@
     QString       record_file_prefix;
     size_t        desired_space;
     uint          desired_freq;
+    size_t        max_record_rate; // bytes/sec
     bool          expire_thread_running;
     bool          is_master_backend;
 
+    // Pending truncates monitor
+    mutable QMutex truncate_monitor_lock;
+    QWaitCondition truncate_monitor_condition;
+    int            truncates_pending;
+
     // update info
     bool          update_pending;
     pthread_t     update_thread;