MythTV  master
eitcache.cpp
Go to the documentation of this file.
1 // -*- Mode: c++ -*-
2 /*
3  * Copyright 2006 (C) Stuart Auchterlonie <stuarta at squashedfrog.net>
4  * Copyright 2006 (C) Janne Grunau <janne-mythtv at grunau.be>
5  * License: GPL v2
6  */
7 
8 #include <QDateTime>
9 
10 #include "eitcache.h"
11 #include "mythcontext.h"
12 #include "mythdb.h"
13 #include "mythlogging.h"
14 #include "mythdate.h"
15 
16 #define LOC QString("EITCache: ")
17 
18 // Highest version number. version is 5bits
19 const uint EITCache::kVersionMax = 31;
20 
22 {
23  // 24 hours ago
24 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
25  m_lastPruneTime = MythDate::current().toUTC().toTime_t() - 86400;
26 #else
27  m_lastPruneTime = MythDate::current().toUTC().toSecsSinceEpoch() - 86400;
28 #endif
29 }
30 
32 {
33  WriteToDB();
34 }
35 
37 {
38  m_accessCnt = 0;
39  m_hitCnt = 0;
40  m_tblChgCnt = 0;
41  m_verChgCnt = 0;
42  m_endChgCnt = 0;
43  m_entryCnt = 0;
44  m_pruneCnt = 0;
45  m_prunedHitCnt = 0;
46  m_futureHitCnt = 0;
48 }
49 
50 QString EITCache::GetStatistics(void) const
51 {
52  QMutexLocker locker(&m_eventMapLock);
53  return QString(
54  "EITCache stats: Access:%1 Hits:%2 "
55  "Table:%3 Version:%4 Endtime:%5 New:%6 "
56  "Pruned:%7 Pruned Hits:%8 Future:%9 Wrong Channel:%10 "
57  "Hit Ratio:%11")
58  .arg(m_accessCnt).arg(m_hitCnt)
59  .arg(m_tblChgCnt).arg(m_verChgCnt).arg(m_endChgCnt).arg(m_entryCnt)
62 }
63 
64 /*
65  * FIXME: This code has a builtin assumption that all timestamps will
66  * fit into a 32bit integer. Qt5.8 has switched to using a 64bit
67  * integer for timestamps.
68  */
69 static inline uint64_t construct_sig(uint tableid, uint version,
70  uint endtime, bool modified)
71 {
72  return (((uint64_t) modified << 63) | ((uint64_t) tableid << 40) |
73  ((uint64_t) version << 32) | ((uint64_t) endtime));
74 }
75 
76 static inline uint extract_table_id(uint64_t sig)
77 {
78  return (sig >> 40) & 0xff;
79 }
80 
81 static inline uint extract_version(uint64_t sig)
82 {
83  return (sig >> 32) & 0x1f;
84 }
85 
86 static inline uint extract_endtime(uint64_t sig)
87 {
88  return sig & 0xffffffff;
89 }
90 
91 static inline bool modified(uint64_t sig)
92 {
93  return (sig >> 63) != 0U;
94 }
95 
96 static void replace_in_db(QStringList &value_clauses,
97  uint chanid, uint eventid, uint64_t sig)
98 {
99  value_clauses << QString("(%1,%2,%3,%4,%5)")
100  .arg(chanid).arg(eventid).arg(extract_table_id(sig))
101  .arg(extract_version(sig)).arg(extract_endtime(sig));
102 }
103 
104 static void delete_in_db(uint endtime)
105 {
106  LOG(VB_EIT, LOG_INFO, LOC + "Deleting old cache entries from the database");
108 
109  QString qstr =
110  "DELETE FROM eit_cache "
111  "WHERE endtime < :ENDTIME";
112 
113  query.prepare(qstr);
114  query.bindValue(":ENDTIME", endtime);
115 
116  if (!query.exec())
117  MythDB::DBError("Error deleting old eitcache entries.", query);
118 }
119 
120 
121 #define EITDATA 0
122 #define CHANNEL_LOCK 1
123 #define STATISTIC 2
124 
125 static bool lock_channel(uint chanid, uint endtime)
126 {
127  int lock = 1;
129 
130  QString qstr = "SELECT COUNT(*) "
131  "FROM eit_cache "
132  "WHERE chanid = :CHANID AND "
133  " endtime > :ENDTIME AND "
134  " status = :STATUS";
135 
136  query.prepare(qstr);
137  query.bindValue(":CHANID", chanid);
138  query.bindValue(":ENDTIME", endtime);
139  query.bindValue(":STATUS", CHANNEL_LOCK);
140 
141  if (!query.exec() || !query.isActive())
142  {
143  MythDB::DBError("Error checking for channel lock", query);
144  return false;
145  }
146 
147  if (query.next())
148  lock = query.value(0).toInt();
149 
150  if (lock)
151  {
152  LOG(VB_EIT, LOG_INFO,
153  LOC + QString("Ignoring channel %1 since it is locked.")
154  .arg(chanid));
155  return false;
156  }
157 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
158  uint now = MythDate::current().toTime_t();
159 #else
160  uint now = MythDate::current().toSecsSinceEpoch();
161 #endif
162  qstr = "INSERT INTO eit_cache "
163  " ( chanid, endtime, status) "
164  "VALUES (:CHANID, :ENDTIME, :STATUS)";
165 
166  query.prepare(qstr);
167  query.bindValue(":CHANID", chanid);
168  query.bindValue(":ENDTIME", now);
169  query.bindValue(":STATUS", CHANNEL_LOCK);
170 
171  if (!query.exec())
172  {
173  MythDB::DBError("Error inserting channel lock", query);
174  return false;
175  }
176 
177  return true;
178 }
179 
180 static void unlock_channel(uint chanid, uint updated)
181 {
183 
184  QString qstr =
185  "DELETE FROM eit_cache "
186  "WHERE chanid = :CHANID AND "
187  " status = :STATUS";
188 
189  query.prepare(qstr);
190  query.bindValue(":CHANID", chanid);
191  query.bindValue(":STATUS", CHANNEL_LOCK);
192 
193  if (!query.exec())
194  MythDB::DBError("Error deleting channel lock", query);
195 
196  // inserting statistics
197 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
198  uint now = MythDate::current().toTime_t();
199 #else
200  uint now = MythDate::current().toSecsSinceEpoch();
201 #endif
202  qstr = "REPLACE INTO eit_cache "
203  " ( chanid, eventid, endtime, status) "
204  "VALUES (:CHANID, :EVENTID, :ENDTIME, :STATUS)";
205 
206  query.prepare(qstr);
207  query.bindValue(":CHANID", chanid);
208  query.bindValue(":EVENTID", updated);
209  query.bindValue(":ENDTIME", now);
210  query.bindValue(":STATUS", STATISTIC);
211 
212  if (!query.exec())
213  MythDB::DBError("Error inserting eit statistics", query);
214 }
215 
216 
218 {
219  if (!lock_channel(chanid, m_lastPruneTime))
220  return nullptr;
221 
223 
224  QString qstr =
225  "SELECT eventid,tableid,version,endtime "
226  "FROM eit_cache "
227  "WHERE chanid = :CHANID AND "
228  " endtime > :ENDTIME AND "
229  " status = :STATUS";
230 
231  query.prepare(qstr);
232  query.bindValue(":CHANID", chanid);
233  query.bindValue(":ENDTIME", m_lastPruneTime);
234  query.bindValue(":STATUS", EITDATA);
235 
236  if (!query.exec() || !query.isActive())
237  {
238  MythDB::DBError("Error loading eitcache", query);
239  return nullptr;
240  }
241 
242  auto *eventMap = new event_map_t();
243 
244  while (query.next())
245  {
246  uint eventid = query.value(0).toUInt();
247  uint tableid = query.value(1).toUInt();
248  uint version = query.value(2).toUInt();
249  uint endtime = query.value(3).toUInt();
250 
251  (*eventMap)[eventid] = construct_sig(tableid, version, endtime, false);
252  }
253 
254  if (!eventMap->empty())
255  LOG(VB_EIT, LOG_INFO, LOC + QString("Loaded %1 entries for channel %2")
256  .arg(eventMap->size()).arg(chanid));
257 
258  m_entryCnt += eventMap->size();
259  return eventMap;
260 }
261 
262 bool EITCache::WriteChannelToDB(QStringList &value_clauses, uint chanid)
263 {
264  event_map_t * eventMap = m_channelMap[chanid];
265 
266  if (!eventMap)
267  return false;
268 
269  uint size = eventMap->size();
270  uint updated = 0;
271  uint removed = 0;
272 
273  event_map_t::iterator it = eventMap->begin();
274  while (it != eventMap->end())
275  {
276  if (extract_endtime(*it) > m_lastPruneTime)
277  {
278  if (modified(*it))
279  {
280  replace_in_db(value_clauses, chanid, it.key(), *it);
281  updated++;
282  *it &= ~(uint64_t)0 >> 1; // mark as synced
283  }
284  ++it;
285  }
286  else
287  {
288  // Event is too old; remove from eit cache in memory
289  it = eventMap->erase(it);
290  removed++;
291  }
292  }
293  unlock_channel(chanid, updated);
294 
295  if (updated)
296  {
297  LOG(VB_EIT, LOG_INFO, LOC + QString("Writing %1 modified entries of %2 "
298  "for channel %3 to database.")
299  .arg(updated).arg(size).arg(chanid));
300  }
301  if (removed)
302  {
303  LOG(VB_EIT, LOG_INFO, LOC + QString("Removed %1 old entries of %2 "
304  "for channel %3 from cache.")
305  .arg(removed).arg(size).arg(chanid));
306  }
307  m_pruneCnt += removed;
308 
309  return true;
310 }
311 
313 {
314  QMutexLocker locker(&m_eventMapLock);
315 
316  QStringList value_clauses;
317  key_map_t::iterator it = m_channelMap.begin();
318  while (it != m_channelMap.end())
319  {
320  if (!WriteChannelToDB(value_clauses, it.key()))
321  it = m_channelMap.erase(it);
322  else
323  ++it;
324  }
325 
326  if(value_clauses.isEmpty())
327  {
328  return;
329  }
330 
332  query.prepare(QString("REPLACE INTO eit_cache "
333  "(chanid, eventid, tableid, version, endtime) "
334  "VALUES %1").arg(value_clauses.join(",")));
335  if (!query.exec())
336  {
337  MythDB::DBError("Error updating eitcache", query);
338  }
339 }
340 
341 bool EITCache::IsNewEIT(uint chanid, uint tableid, uint version,
342  uint eventid, uint endtime)
343 {
344  m_accessCnt++;
345 
346  if (m_accessCnt % 500000 == 50000)
347  {
348  LOG(VB_EIT, LOG_INFO, GetStatistics());
349  WriteToDB();
350  }
351 
352  // don't re-add pruned entries
353  if (endtime < m_lastPruneTime)
354  {
355  m_prunedHitCnt++;
356  return false;
357  }
358 
359  // validity check, reject events with endtime over 7 weeks in the future
360  if (endtime > m_lastPruneTime + 50 * 86400)
361  {
362  m_futureHitCnt++;
363  return false;
364  }
365 
366  QMutexLocker locker(&m_eventMapLock);
367  if (!m_channelMap.contains(chanid))
368  {
369  m_channelMap[chanid] = LoadChannel(chanid);
370  }
371 
372  if (!m_channelMap[chanid])
373  {
375  return false;
376  }
377 
378  event_map_t * eventMap = m_channelMap[chanid];
379  event_map_t::iterator it = eventMap->find(eventid);
380  if (it != eventMap->end())
381  {
382  if (extract_table_id(*it) > tableid)
383  {
384  // EIT from lower (ie. better) table number
385  m_tblChgCnt++;
386  }
387  else if ((extract_table_id(*it) == tableid) &&
388  (extract_version(*it) != version))
389  {
390  // EIT updated version on current table
391  m_verChgCnt++;
392  }
393  else if (extract_endtime(*it) != endtime)
394  {
395  // Endtime (starttime + duration) changed
396  m_endChgCnt++;
397  }
398  else
399  {
400  // EIT data previously seen
401  m_hitCnt++;
402  return false;
403  }
404  }
405 
406  eventMap->insert(eventid, construct_sig(tableid, version, endtime, true));
407  m_entryCnt++;
408 
409  return true;
410 }
411 
417 {
418  if (VERBOSE_LEVEL_CHECK(VB_EIT, LOG_INFO))
419  {
420 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
421  QDateTime tmptime = MythDate::fromTime_t(timestamp);
422 #else
423  QDateTime tmptime = MythDate::fromSecsSinceEpoch(timestamp);
424 #endif
425  LOG(VB_EIT, LOG_INFO,
426  LOC + "Pruning all entries that ended before UTC " +
427  tmptime.toString(Qt::ISODate));
428  }
429 
430  m_lastPruneTime = timestamp;
431 
432  // Write all modified entries to DB and start with a clean cache
433  WriteToDB();
434 
435  // Prune old entries in the DB
436  delete_in_db(timestamp);
437 
438  return 0;
439 }
440 
441 
446 {
448 
449  QString qstr =
450  "DELETE FROM eit_cache "
451  "WHERE status = :STATUS";
452 
453  query.prepare(qstr);
454  query.bindValue(":STATUS", CHANNEL_LOCK);
455 
456  if (!query.exec())
457  MythDB::DBError("Error clearing channel locks", query);
458 }
459 
460 /* vim: set expandtab tabstop=4 shiftwidth=4: */
bool next(void)
Wrap QSqlQuery::next() so we can display the query results.
Definition: mythdbcon.cpp:783
void bindValue(const QString &placeholder, const QVariant &val)
Add a single binding.
Definition: mythdbcon.cpp:864
#define LOC
Definition: eitcache.cpp:16
#define CHANNEL_LOCK
Definition: eitcache.cpp:122
static uint extract_endtime(uint64_t sig)
Definition: eitcache.cpp:86
uint m_pruneCnt
Definition: eitcache.h:54
static MTV_PUBLIC void ClearChannelLocks(void)
removes old channel locks, use it only at master backend start
Definition: eitcache.cpp:445
uint m_prunedHitCnt
Definition: eitcache.h:55
static uint extract_table_id(uint64_t sig)
Definition: eitcache.cpp:76
QSqlQuery wrapper that fetches a DB connection from the connection pool.
Definition: mythdbcon.h:125
uint m_verChgCnt
Definition: eitcache.h:51
static void replace_in_db(QStringList &value_clauses, uint chanid, uint eventid, uint64_t sig)
Definition: eitcache.cpp:96
key_map_t m_channelMap
Definition: eitcache.h:42
uint m_tblChgCnt
Definition: eitcache.h:50
uint m_hitCnt
Definition: eitcache.h:49
QVariant value(int i) const
Definition: mythdbcon.h:198
uint m_wrongChannelHitCnt
Definition: eitcache.h:57
void ResetStatistics(void)
Definition: eitcache.cpp:36
uint m_futureHitCnt
Definition: eitcache.h:56
uint m_lastPruneTime
Definition: eitcache.h:45
MBASE_PUBLIC QDateTime fromSecsSinceEpoch(uint seconds)
This function takes the number of seconds since the start of the epoch and returns a QDateTime with t...
Definition: mythdate.cpp:88
MSqlQuery query(MSqlQuery::InitCon())
#define VERBOSE_LEVEL_CHECK(_MASK_, _LEVEL_)
Definition: mythlogging.h:14
static uint64_t construct_sig(uint tableid, uint version, uint endtime, bool modified)
Definition: eitcache.cpp:69
void WriteToDB(void)
Definition: eitcache.cpp:312
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
EITCache()
Definition: eitcache.cpp:21
static bool lock_channel(uint chanid, uint endtime)
Definition: eitcache.cpp:125
QMap< uint, uint64_t > event_map_t
Definition: eitcache.h:19
bool isActive(void) const
Definition: mythdbcon.h:204
unsigned int uint
Definition: compat.h:140
QString GetStatistics(void) const
Definition: eitcache.cpp:50
static MSqlQueryInfo InitCon(ConnectionReuse _reuse=kNormalConnection)
Only use this in combination with MSqlQuery constructor.
Definition: mythdbcon.cpp:535
#define EITDATA
Definition: eitcache.cpp:121
uint PruneOldEntries(uint utc_timestamp)
Prunes entries that describe events ending before timestamp time.
Definition: eitcache.cpp:416
QMutex m_eventMapLock
Definition: eitcache.h:44
bool WriteChannelToDB(QStringList &value_clauses, uint chanid)
Definition: eitcache.cpp:262
static void delete_in_db(uint endtime)
Definition: eitcache.cpp:104
bool prepare(const QString &query)
QSqlQuery::prepare() is not thread safe in Qt <= 3.3.2.
Definition: mythdbcon.cpp:808
static void unlock_channel(uint chanid, uint updated)
Definition: eitcache.cpp:180
static const uint kVersionMax
Definition: eitcache.h:59
bool IsNewEIT(uint chanid, uint tableid, uint version, uint eventid, uint endtime)
Definition: eitcache.cpp:341
uint m_endChgCnt
Definition: eitcache.h:52
static uint extract_version(uint64_t sig)
Definition: eitcache.cpp:81
static bool modified(uint64_t sig)
Definition: eitcache.cpp:91
uint m_entryCnt
Definition: eitcache.h:53
uint m_accessCnt
Definition: eitcache.h:48
Default UTC.
Definition: mythdate.h:14
bool exec(void)
Wrap QSqlQuery::exec() so we can display SQL.
Definition: mythdbcon.cpp:603
static void DBError(const QString &where, const MSqlQuery &query)
Definition: mythdb.cpp:179
~EITCache()
Definition: eitcache.cpp:31
#define STATISTIC
Definition: eitcache.cpp:123
event_map_t * LoadChannel(uint chanid)
Definition: eitcache.cpp:217
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23