Ticket #1728: eitcache_db.2.diff

libs/libmythtv/dbcheck.cpp

@@ -10 +10 @@
 #include "mythdbcon.h"
 
 /// This is the DB schema version expected by the running MythTV instance.
-const QString currentDatabaseVersion = "1138";
+const QString currentDatabaseVersion = "1139";
 
 static bool UpdateDBVersionNumber(const QString &newnumber);
 static bool performActualUpdate(const QString updates[], QString version,
@@ -2213 +2213 @@
             return false;
     }
 
+    if (dbver == "1138")
+    {
+        const QString updates[] = {
+"CREATE TABLE eit_cache ("
+"  chanid  INT(10) NOT NULL, "
+"  eventid SMALLINT UNSIGNED NOT NULL, "
+"  tableid TINYINT UNSIGNED NOT NULL, "
+"  version TINYINT UNSIGNED NOT NULL, "
+"  endtime INT UNSIGNED NOT NULL, "
+"  PRIMARY KEY  (chanid, eventid) "
+");",
+"",
+};
+
+       if (!performActualUpdate(updates, "1139", dbver))
+            return false;
+    }
+
 //"ALTER TABLE capturecard DROP COLUMN dvb_recordts;" in 0.21
 //"ALTER TABLE capturecard DROP COLUMN dvb_hw_decoder;" in 0.21
 //"ALTER TABLE cardinput DROP COLUMN  preference;" in 0.22

libs/libmythtv/eitcache.cpp

@@ -10 +10 @@
 
 #include "eitcache.h"
 #include "mythcontext.h"
+#include "mythdbcon.h"
 
 #define LOC QString("EITCache: ")
 
@@ -17 +18 @@
 const uint EITCache::kVersionMax = 31;
 
 EITCache::EITCache()
-    : accessCnt(0), hitCnt(0),   tblChgCnt(0),
-      verChgCnt(0), pruneCnt(0), prunedHitCnt(0)
+    : accessCnt(0), hitCnt(0),   tblChgCnt(0),   verChgCnt(0),
+      entryCnt(0), pruneCnt(0), prunedHitCnt(0), wrongChannelHitCnt(0)
 {
     // 24 hours ago
     lastPruneTime = QDateTime::currentDateTime(Qt::UTC).toTime_t() - 86400;
 }
 
+EITCache::~EITCache()
+{
+    WriteToDB();
+}
+
 void EITCache::ResetStatistics(void)
 {
     accessCnt = 0;
     hitCnt    = 0;
     tblChgCnt = 0;
     verChgCnt = 0;
+    entryCnt  = 0;
     pruneCnt  = 0;
     prunedHitCnt = 0;
+    wrongChannelHitCnt = 0;
 }
 
 QString EITCache::GetStatistics(void) const
@@ -40 +48 @@
     return QString(
         "EITCache::statistics: Accesses: %1, Hits: %2, "
         "Table Upgrades %3, New Versions: %4, Entries: %5 "
-        "Pruned entries: %6, pruned Hits: %7.")
+        "Pruned entries: %6, pruned Hits: %7 Discard channel Hit %8 "
+        "Hit Ratio %9.")
         .arg(accessCnt).arg(hitCnt).arg(tblChgCnt).arg(verChgCnt)
-        .arg(eventMap.size()).arg(pruneCnt).arg(prunedHitCnt);
+        .arg(entryCnt).arg(pruneCnt).arg(prunedHitCnt)
+        .arg(wrongChannelHitCnt)
+        .arg((hitCnt+prunedHitCnt+wrongChannelHitCnt)/(double)accessCnt);
+}
+
+static inline uint64_t construct_channel(uint networkid, uint tsid, uint serviceid)
+{
+    return (((uint64_t) networkid << 32) | ((uint64_t) tsid    << 16) |
+            ((uint64_t) serviceid));
+}
+
+static inline uint extract_onid(uint64_t channel)
+{
+    return (channel >> 32) & 0xffff;
+}
+
+static inline uint extract_tsid(uint64_t channel)
+{
+    return (channel >> 16) & 0xffff;
 }
 
-static uint64_t construct_key(uint networkid, uint tsid,
-                              uint serviceid, uint eventid)
+static inline uint extrac_sid(uint64_t channel)
 {
-    return (((uint64_t) networkid << 48) | ((uint64_t) tsid    << 32) |
-            ((uint64_t) serviceid << 16) | ((uint64_t) eventid      ));
+    return channel & 0xffff;
 }
 
-static uint64_t construct_sig(uint tableid, uint version, uint endtime)
+static inline uint64_t construct_sig(uint tableid, uint version, uint endtime,
+                              bool modified)
 {
-    return (((uint64_t) tableid   << 40) | ((uint64_t) version   << 32) |
-            ((uint64_t) endtime));
+    return (((uint64_t) modified  << 63) | ((uint64_t) tableid   << 40) |
+            ((uint64_t) version   << 32) | ((uint64_t) endtime));
 }
 
-static uint extract_table_id(uint64_t sig)
+static inline uint extract_table_id(uint64_t sig)
 {
     return (sig >> 40) & 0xff;
 }
 
-static uint extract_version(uint64_t sig)
+static inline uint extract_version(uint64_t sig)
 {
     return (sig >> 32) & 0x1f;
 }
 
-static uint extract_endtime(uint64_t sig)
+static inline uint extract_endtime(uint64_t sig)
 {
     return sig & 0xffffffff;
 }
 
+static inline bool modified(uint64_t sig)
+{
+    return sig >> 63;
+}
+
+static int get_chanid_from_db(uint networkid, uint tsid, uint serviceid)
+{
+    MSqlQuery query(MSqlQuery::InitCon());
+
+    // DVB Link to chanid
+    QString qstr =
+        "SELECT chanid, useonairguide "
+        "FROM channel, dtv_multiplex "
+        "WHERE serviceid        = :SERVICEID   AND "
+        "      networkid        = :NETWORKID   AND "
+        "      transportid      = :TRANSPORTID AND "
+        "      channel.mplexid  = dtv_multiplex.mplexid";
+
+    query.prepare(qstr);
+    query.bindValue(":SERVICEID",   serviceid);
+    query.bindValue(":NETWORKID",   networkid);
+    query.bindValue(":TRANSPORTID", tsid);
+
+    if (!query.exec() || !query.isActive())
+        MythContext::DBError("Looking up chanID", query);
+    else if (query.next())
+    {
+        bool useOnAirGuide = query.value(1).toBool();
+        return (useOnAirGuide) ? query.value(0).toInt() : -1;
+    }
+    return -1;
+}
+
+static void replace_in_db(int chanid, uint eventid, uint64_t sig)
+{
+
+    MSqlQuery query(MSqlQuery::InitCon());
+
+    QString qstr =
+        "REPLACE INTO eit_cache "
+        "       ( chanid,  eventid,  tableid,  version,  endtime) "
+        "VALUES (:CHANID, :EVENTID, :TABLEID, :VERSION, :ENDTIME)";
+
+    query.prepare(qstr);
+    query.bindValue(":CHANID",   chanid);
+    query.bindValue(":EVENTID",  eventid);
+    query.bindValue(":TABLEID",  extract_table_id(sig));
+    query.bindValue(":VERSION",  extract_version(sig));
+    query.bindValue(":ENDTIME",  extract_endtime(sig));
+
+    if (!query.exec())
+        MythContext::DBError("Error updating eitcache", query);
+
+    return;
+}
+
+
+event_map_t * EITCache::LoadChannel(uint networkid, uint tsid, uint serviceid)
+{
+    int chanid = get_chanid_from_db(networkid, tsid, serviceid);
+
+    if (chanid < 1)
+        return NULL;
+
+    MSqlQuery query(MSqlQuery::InitCon());
+
+    QString qstr =
+        "SELECT eventid,tableid,version,endtime "
+        "FROM eit_cache "
+        "WHERE chanid        = :CHANID   AND "
+        "      endtime       > :ENDTIME";
+
+    query.prepare(qstr);
+    query.bindValue(":CHANID",   chanid);
+    query.bindValue(":ENDTIME",  lastPruneTime);
+
+    if (!query.exec() || !query.isActive())
+    {
+        MythContext::DBError("Error loading eitcache", query);
+        return NULL;
+    }
+
+    event_map_t * eventMap = new event_map_t();
+
+    while (query.next())
+    {
+        uint eventid = query.value(0).toUInt();
+        uint tableid = query.value(1).toUInt();
+        uint version = query.value(2).toUInt();
+        uint endtime = query.value(3).toUInt();
+
+        (*eventMap)[eventid] = construct_sig(tableid, version, endtime, false);
+    }
+
+    VERBOSE(VB_EIT, LOC + QString("Loaded %1 entries for channel %2")
+            .arg(eventMap->size()).arg(chanid));
+
+    entryCnt += eventMap->size();
+    return eventMap;
+}
+
+
+void EITCache::DropChannel(uint64_t channel)
+{
+    int chanid = get_chanid_from_db(extract_onid(channel),
+                                    extract_tsid(channel),
+                                    extrac_sid(channel));
+
+    if (chanid < 1)
+        return;
+
+    event_map_t * eventMap = channelMap[channel];
+
+    uint size    = eventMap->size();
+    uint written = 0;
+
+    event_map_t::iterator it = eventMap->begin();
+    while (it != eventMap->end())
+    {
+        if (modified(*it) && extract_endtime(*it) > lastPruneTime)
+        {
+            replace_in_db(chanid, it.key(), *it);
+            written++;
+        }
+
+        event_map_t::iterator tmp = it;
+        ++tmp;
+        eventMap->erase(it);
+        it = tmp;
+    }
+    channelMap.erase(channel);
+
+    VERBOSE(VB_EIT, LOC + QString("Writed %1 modified entries of %2 "
+                                  "for channel %3 to database.")
+            .arg(written).arg(size).arg(chanid));
+    entryCnt -= size;
+}
+
+void EITCache::WriteToDB(void)
+{
+    QMutexLocker locker(&eventMapLock);
+
+    key_map_t::iterator it = channelMap.begin();
+    while (it != channelMap.end())
+    {
+        key_map_t::iterator tmp = it;
+        ++tmp;
+        DropChannel(it.key());
+        it = tmp;
+    }
+    entryCnt = 0;
+}
+
+
+
 bool EITCache::IsNewEIT(uint networkid, uint tsid,    uint serviceid,
                         uint tableid,   uint version,
                         uint eventid,   uint endtime)
 {
     accessCnt++;
 
+    if (accessCnt % 500000 == 50000)
+    {
+        cerr << GetStatistics() << endl;
+        WriteToDB();
+    }
+
     // don't readd pruned entries
     if (endtime < lastPruneTime)
     {
@@ -86 +273 @@
         return false;
     }
 
-    uint64_t key = construct_key(networkid, tsid, serviceid, eventid);
+    uint64_t channel = construct_channel(networkid, tsid, serviceid);
 
     QMutexLocker locker(&eventMapLock);
-    key_map_t::const_iterator it = eventMap.find(key);
+    if (!channelMap.contains(channel))
+    {
+        channelMap[channel] = LoadChannel(networkid, tsid, serviceid);
+    }
 
-    if (it != eventMap.end())
+    if (!channelMap[channel])
+    {
+        wrongChannelHitCnt++;
+        return false;
+    }
+
+    event_map_t * eventMap = channelMap[channel];
+    event_map_t::iterator it = eventMap->find(eventid);
+    if (it != eventMap->end())
     {
         if (extract_table_id(*it) > tableid)
         {
@@ -113 +311 @@
             return false;
         }
     }
-
-    eventMap[key] = construct_sig(tableid, version, endtime);
+    eventMap->insert(eventid, construct_sig(tableid, version, endtime, true));
+    entryCnt++;
 
     return true;
 }
@@ -133 +331 @@
                 tmptime.toString(Qt::ISODate));
     }
 
-    QMutexLocker locker(&eventMapLock);
-    uint orig_size = eventMap.size();
-
-    key_map_t::iterator it = eventMap.begin();
-    while (it != eventMap.end())
-    {
-        if (extract_endtime(*it) < timestamp)
-        {
-            key_map_t::iterator tmp = it;
-            ++tmp;
-            eventMap.erase(it);
-            it = tmp;
-            continue;
-        }
-        ++it;
-    }
-
-    uint pruned    = orig_size - eventMap.size();
-    prunedHitCnt  += pruned;
     lastPruneTime  = timestamp;
+    //Write all modified entries to DB and start with a clean cache
+    WriteToDB();
+    // TODO: prune old entries in the DB
 
-    return pruned;
+    return 0;
 }
 
 /* vim: set expandtab tabstop=4 shiftwidth=4: */

libs/libmythtv/eitcache.h

@@ -13 +13 @@
 #include <qmutex.h>
 #include <qstring.h>
 
-typedef QMap<uint64_t, uint64_t> key_map_t;
+typedef QMap<uint, uint64_t> event_map_t;
+typedef QMap<uint64_t, event_map_t*> key_map_t;
 
 class EITCache
 {
   public:
     EITCache();
-   ~EITCache() {};
+   ~EITCache();
 
     bool IsNewEIT(uint networkid, uint tsid,    uint serviceid,
                   uint tableid,   uint version,
@@ -31 +32 @@
     QString GetStatistics(void) const;
 
   private:
+    event_map_t * LoadChannel(uint networkid, uint tsid, uint serviceid);
+    void DropChannel(uint64_t channel);
+    void WriteToDB(void);
+
     // event key cache
-    key_map_t   eventMap;
+    key_map_t   channelMap;
 
     mutable QMutex eventMapLock;
     uint            lastPruneTime;
@@ -42 +47 @@
     uint        hitCnt;
     uint        tblChgCnt;
     uint        verChgCnt;
+    uint        entryCnt;
     uint        pruneCnt;
     uint        prunedHitCnt;
+    uint        wrongChannelHitCnt;
 
     static const uint kVersionMax;
 };

libs/libmythtv/eithelper.h

@@ -69 +69 @@
 
     void AddEIT(const DVBEventInformationTable *eit);
 
+    void PruneCache(uint timestamp);
+
   private:
     uint GetChanID(uint atscsrcid);
     uint GetChanID(uint serviceid, uint networkid, uint transportid);

libs/libmythtv/eitscanner.cpp

@@ -135 +135 @@
             activeScanNextTrig = QDateTime::currentDateTime()
                 .addSecs(activeScanTrigTime);
             activeScanNextChan++;
+
+            // 24 hours ago
+            eitHelper->PruneCache(activeScanNextTrig.toTime_t() - 86400);
         }
 
         exitThreadCond.wait(200); // sleep up to 200 ms.

libs/libmythtv/eithelper.cpp

@@ -336 +336 @@
     }
 }
 
+void EITHelper::PruneCache(uint timestamp)
+{
+    eitcache->PruneOldEntries(timestamp);
+}
+
 //////////////////////////////////////////////////////////////////////
 // private methods and functions below this line                    //
 //////////////////////////////////////////////////////////////////////