MythTV  master
eitcache.cpp
Go to the documentation of this file.
1 // -*- Mode: c++ -*-
2 /*
3  * Copyright 2006 (C) Stuart Auchterlonie <stuarta at squashedfrog.net>
4  * Copyright 2006 (C) Janne Grunau <janne-mythtv at grunau.be>
5  * License: GPL v2
6  */
7 
8 #include <QDateTime>
9 
10 #include "eitcache.h"
11 #include "mythcontext.h"
12 #include "mythdb.h"
13 #include "mythlogging.h"
14 #include "mythdate.h"
15 
16 #define LOC QString("EITCache: ")
17 
18 // Highest version number. version is 5bits
19 const uint EITCache::kVersionMax = 31;
20 
22 {
23  // 24 hours ago
24 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
25  m_lastPruneTime = MythDate::current().toUTC().toTime_t() - 86400;
26 #else
27  m_lastPruneTime = MythDate::current().toUTC().toSecsSinceEpoch() - 86400;
28 #endif
29 }
30 
32 {
33  WriteToDB();
34 }
35 
37 {
38  m_accessCnt = 0;
39  m_hitCnt = 0;
40  m_tblChgCnt = 0;
41  m_verChgCnt = 0;
42  m_endChgCnt = 0;
43  m_entryCnt = 0;
44  m_pruneCnt = 0;
45  m_prunedHitCnt = 0;
46  m_futureHitCnt = 0;
48 }
49 
50 QString EITCache::GetStatistics(void) const
51 {
52  QMutexLocker locker(&m_eventMapLock);
53  return QString(
54  "EITCache stats: Access:%1 Hits:%2 "
55  "Table:%3 Version:%4 Endtime:%5 New:%6 "
56  "Pruned:%7 Pruned Hits:%8 Future:%9 Wrong Channel:%10 "
57  "Hit Ratio:%11")
58  .arg(m_accessCnt).arg(m_hitCnt)
59  .arg(m_tblChgCnt).arg(m_verChgCnt).arg(m_endChgCnt).arg(m_entryCnt)
62 }
63 
64 /*
65  * FIXME: This code has a builtin assumption that all timestamps will
66  * fit into a 32bit integer. Qt5.8 has switched to using a 64bit
67  * integer for timestamps.
68  */
69 static inline uint64_t construct_sig(uint tableid, uint version,
70  uint endtime, bool modified)
71 {
72  return (((uint64_t) modified << 63) | ((uint64_t) tableid << 40) |
73  ((uint64_t) version << 32) | ((uint64_t) endtime));
74 }
75 
76 static inline uint extract_table_id(uint64_t sig)
77 {
78  return (sig >> 40) & 0xff;
79 }
80 
81 static inline uint extract_version(uint64_t sig)
82 {
83  return (sig >> 32) & 0x1f;
84 }
85 
86 static inline uint extract_endtime(uint64_t sig)
87 {
88  return sig & 0xffffffff;
89 }
90 
91 static inline bool modified(uint64_t sig)
92 {
93  return (sig >> 63) != 0U;
94 }
95 
96 static void replace_in_db(QStringList &value_clauses,
97  uint chanid, uint eventid, uint64_t sig)
98 {
99  value_clauses << QString("(%1,%2,%3,%4,%5)")
100  .arg(chanid).arg(eventid).arg(extract_table_id(sig))
101  .arg(extract_version(sig)).arg(extract_endtime(sig));
102 }
103 
104 static void delete_in_db(uint endtime)
105 {
106  LOG(VB_EIT, LOG_INFO, LOC + "Deleting old cache entries from the database");
107  MSqlQuery query(MSqlQuery::InitCon());
108 
109  QString qstr =
110  "DELETE FROM eit_cache "
111  "WHERE endtime < :ENDTIME";
112 
113  query.prepare(qstr);
114  query.bindValue(":ENDTIME", endtime);
115 
116  if (!query.exec())
117  MythDB::DBError("Error deleting old eitcache entries.", query);
118 }
119 
120 
121 #define EITDATA 0
122 #define CHANNEL_LOCK 1
123 #define STATISTIC 2
124 
125 static bool lock_channel(uint chanid, uint endtime)
126 {
127  int lock = 1;
128  MSqlQuery query(MSqlQuery::InitCon());
129 
130  QString qstr = "SELECT COUNT(*) "
131  "FROM eit_cache "
132  "WHERE chanid = :CHANID AND "
133  " endtime > :ENDTIME AND "
134  " status = :STATUS";
135 
136  query.prepare(qstr);
137  query.bindValue(":CHANID", chanid);
138  query.bindValue(":ENDTIME", endtime);
139  query.bindValue(":STATUS", CHANNEL_LOCK);
140 
141  if (!query.exec() || !query.isActive())
142  {
143  MythDB::DBError("Error checking for channel lock", query);
144  return false;
145  }
146 
147  if (query.next())
148  lock = query.value(0).toInt();
149 
150  if (lock)
151  {
152  LOG(VB_EIT, LOG_INFO,
153  LOC + QString("Ignoring channel %1 since it is locked.")
154  .arg(chanid));
155  return false;
156  }
157 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
158  uint now = MythDate::current().toTime_t();
159 #else
160  uint now = MythDate::current().toSecsSinceEpoch();
161 #endif
162  qstr = "INSERT INTO eit_cache "
163  " ( chanid, endtime, status) "
164  "VALUES (:CHANID, :ENDTIME, :STATUS)";
165 
166  query.prepare(qstr);
167  query.bindValue(":CHANID", chanid);
168  query.bindValue(":ENDTIME", now);
169  query.bindValue(":STATUS", CHANNEL_LOCK);
170 
171  if (!query.exec())
172  {
173  MythDB::DBError("Error inserting channel lock", query);
174  return false;
175  }
176 
177  return true;
178 }
179 
180 static void unlock_channel(uint chanid, uint updated)
181 {
182  MSqlQuery query(MSqlQuery::InitCon());
183 
184  QString qstr =
185  "DELETE FROM eit_cache "
186  "WHERE chanid = :CHANID AND "
187  " status = :STATUS";
188 
189  query.prepare(qstr);
190  query.bindValue(":CHANID", chanid);
191  query.bindValue(":STATUS", CHANNEL_LOCK);
192 
193  if (!query.exec())
194  MythDB::DBError("Error deleting channel lock", query);
195 
196  // inserting statistics
197 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
198  uint now = MythDate::current().toTime_t();
199 #else
200  uint now = MythDate::current().toSecsSinceEpoch();
201 #endif
202  qstr = "REPLACE INTO eit_cache "
203  " ( chanid, eventid, endtime, status) "
204  "VALUES (:CHANID, :EVENTID, :ENDTIME, :STATUS)";
205 
206  query.prepare(qstr);
207  query.bindValue(":CHANID", chanid);
208  query.bindValue(":EVENTID", updated);
209  query.bindValue(":ENDTIME", now);
210  query.bindValue(":STATUS", STATISTIC);
211 
212  if (!query.exec())
213  MythDB::DBError("Error inserting eit statistics", query);
214 }
215 
216 
218 {
219  if (!lock_channel(chanid, m_lastPruneTime))
220  return nullptr;
221 
222  MSqlQuery query(MSqlQuery::InitCon());
223 
224  QString qstr =
225  "SELECT eventid,tableid,version,endtime "
226  "FROM eit_cache "
227  "WHERE chanid = :CHANID AND "
228  " endtime > :ENDTIME AND "
229  " status = :STATUS";
230 
231  query.prepare(qstr);
232  query.bindValue(":CHANID", chanid);
233  query.bindValue(":ENDTIME", m_lastPruneTime);
234  query.bindValue(":STATUS", EITDATA);
235 
236  if (!query.exec() || !query.isActive())
237  {
238  MythDB::DBError("Error loading eitcache", query);
239  return nullptr;
240  }
241 
242  event_map_t * eventMap = new event_map_t();
243 
244  while (query.next())
245  {
246  uint eventid = query.value(0).toUInt();
247  uint tableid = query.value(1).toUInt();
248  uint version = query.value(2).toUInt();
249  uint endtime = query.value(3).toUInt();
250 
251  (*eventMap)[eventid] = construct_sig(tableid, version, endtime, false);
252  }
253 
254  if (!eventMap->empty())
255  LOG(VB_EIT, LOG_INFO, LOC + QString("Loaded %1 entries for channel %2")
256  .arg(eventMap->size()).arg(chanid));
257 
258  m_entryCnt += eventMap->size();
259  return eventMap;
260 }
261 
262 bool EITCache::WriteChannelToDB(QStringList &value_clauses, uint chanid)
263 {
264  event_map_t * eventMap = m_channelMap[chanid];
265 
266  if (!eventMap)
267  return false;
268 
269  uint size = eventMap->size();
270  uint updated = 0;
271  uint removed = 0;
272 
273  event_map_t::iterator it = eventMap->begin();
274  while (it != eventMap->end())
275  {
276  if (extract_endtime(*it) > m_lastPruneTime)
277  {
278  if (modified(*it))
279  {
280  replace_in_db(value_clauses, chanid, it.key(), *it);
281  updated++;
282  *it &= ~(uint64_t)0 >> 1; // mark as synced
283  }
284  ++it;
285  }
286  else
287  {
288  // Event is too old; remove from eit cache in memory
289  it = eventMap->erase(it);
290  removed++;
291  }
292  }
293  unlock_channel(chanid, updated);
294 
295  if (updated)
296  LOG(VB_EIT, LOG_INFO, LOC + QString("Writing %1 modified entries of %2 "
297  "for channel %3 to database.")
298  .arg(updated).arg(size).arg(chanid));
299  if (removed)
300  LOG(VB_EIT, LOG_INFO, LOC + QString("Removed %1 old entries of %2 "
301  "for channel %3 from cache.")
302  .arg(removed).arg(size).arg(chanid));
303  m_pruneCnt += removed;
304 
305  return true;
306 }
307 
309 {
310  QMutexLocker locker(&m_eventMapLock);
311 
312  QStringList value_clauses;
313  key_map_t::iterator it = m_channelMap.begin();
314  while (it != m_channelMap.end())
315  {
316  if (!WriteChannelToDB(value_clauses, it.key()))
317  it = m_channelMap.erase(it);
318  else
319  ++it;
320  }
321 
322  if(value_clauses.isEmpty())
323  {
324  return;
325  }
326 
327  MSqlQuery query(MSqlQuery::InitCon());
328  query.prepare(QString("REPLACE INTO eit_cache "
329  "(chanid, eventid, tableid, version, endtime) "
330  "VALUES %1").arg(value_clauses.join(",")));
331  if (!query.exec())
332  {
333  MythDB::DBError("Error updating eitcache", query);
334  }
335 }
336 
337 bool EITCache::IsNewEIT(uint chanid, uint tableid, uint version,
338  uint eventid, uint endtime)
339 {
340  m_accessCnt++;
341 
342  if (m_accessCnt % 500000 == 50000)
343  {
344  LOG(VB_EIT, LOG_INFO, GetStatistics());
345  WriteToDB();
346  }
347 
348  // don't re-add pruned entries
349  if (endtime < m_lastPruneTime)
350  {
351  m_prunedHitCnt++;
352  return false;
353  }
354 
355  // validity check, reject events with endtime over 7 weeks in the future
356  if (endtime > m_lastPruneTime + 50 * 86400)
357  {
358  m_futureHitCnt++;
359  return false;
360  }
361 
362  QMutexLocker locker(&m_eventMapLock);
363  if (!m_channelMap.contains(chanid))
364  {
365  m_channelMap[chanid] = LoadChannel(chanid);
366  }
367 
368  if (!m_channelMap[chanid])
369  {
371  return false;
372  }
373 
374  event_map_t * eventMap = m_channelMap[chanid];
375  event_map_t::iterator it = eventMap->find(eventid);
376  if (it != eventMap->end())
377  {
378  if (extract_table_id(*it) > tableid)
379  {
380  // EIT from lower (ie. better) table number
381  m_tblChgCnt++;
382  }
383  else if ((extract_table_id(*it) == tableid) &&
384  (extract_version(*it) != version))
385  {
386  // EIT updated version on current table
387  m_verChgCnt++;
388  }
389  else if (extract_endtime(*it) != endtime)
390  {
391  // Endtime (starttime + duration) changed
392  m_endChgCnt++;
393  }
394  else
395  {
396  // EIT data previously seen
397  m_hitCnt++;
398  return false;
399  }
400  }
401 
402  eventMap->insert(eventid, construct_sig(tableid, version, endtime, true));
403  m_entryCnt++;
404 
405  return true;
406 }
407 
413 {
414  if (VERBOSE_LEVEL_CHECK(VB_EIT, LOG_INFO))
415  {
416 #if QT_VERSION < QT_VERSION_CHECK(5,8,0)
417  QDateTime tmptime = MythDate::fromTime_t(timestamp);
418 #else
419  QDateTime tmptime = MythDate::fromSecsSinceEpoch(timestamp);
420 #endif
421  LOG(VB_EIT, LOG_INFO,
422  LOC + "Pruning all entries that ended before UTC " +
423  tmptime.toString(Qt::ISODate));
424  }
425 
426  m_lastPruneTime = timestamp;
427 
428  // Write all modified entries to DB and start with a clean cache
429  WriteToDB();
430 
431  // Prune old entries in the DB
432  delete_in_db(timestamp);
433 
434  return 0;
435 }
436 
437 
442 {
443  MSqlQuery query(MSqlQuery::InitCon());
444 
445  QString qstr =
446  "DELETE FROM eit_cache "
447  "WHERE status = :STATUS";
448 
449  query.prepare(qstr);
450  query.bindValue(":STATUS", CHANNEL_LOCK);
451 
452  if (!query.exec())
453  MythDB::DBError("Error clearing channel locks", query);
454 }
455 
456 /* vim: set expandtab tabstop=4 shiftwidth=4: */
bool next(void)
Wrap QSqlQuery::next() so we can display the query results.
Definition: mythdbcon.cpp:782
void bindValue(const QString &placeholder, const QVariant &val)
Add a single binding.
Definition: mythdbcon.cpp:863
#define LOC
Definition: eitcache.cpp:16
QMap< uint, uint64_t > event_map_t
Definition: eitcache.h:19
#define CHANNEL_LOCK
Definition: eitcache.cpp:122
static uint extract_endtime(uint64_t sig)
Definition: eitcache.cpp:86
uint m_pruneCnt
Definition: eitcache.h:54
static MTV_PUBLIC void ClearChannelLocks(void)
removes old channel locks, use it only at master backend start
Definition: eitcache.cpp:441
uint m_prunedHitCnt
Definition: eitcache.h:55
static uint extract_table_id(uint64_t sig)
Definition: eitcache.cpp:76
QSqlQuery wrapper that fetches a DB connection from the connection pool.
Definition: mythdbcon.h:125
uint m_verChgCnt
Definition: eitcache.h:51
static void replace_in_db(QStringList &value_clauses, uint chanid, uint eventid, uint64_t sig)
Definition: eitcache.cpp:96
key_map_t m_channelMap
Definition: eitcache.h:42
unsigned int uint
Definition: compat.h:140
uint m_tblChgCnt
Definition: eitcache.h:50
uint m_hitCnt
Definition: eitcache.h:49
QVariant value(int i) const
Definition: mythdbcon.h:198
uint m_wrongChannelHitCnt
Definition: eitcache.h:57
void ResetStatistics(void)
Definition: eitcache.cpp:36
uint m_futureHitCnt
Definition: eitcache.h:56
uint m_lastPruneTime
Definition: eitcache.h:45
MBASE_PUBLIC QDateTime fromSecsSinceEpoch(uint seconds)
This function takes the number of seconds since the start of the epoch and returns a QDateTime with t...
Definition: mythdate.cpp:88
#define VERBOSE_LEVEL_CHECK(_MASK_, _LEVEL_)
Definition: mythlogging.h:24
static uint64_t construct_sig(uint tableid, uint version, uint endtime, bool modified)
Definition: eitcache.cpp:69
void WriteToDB(void)
Definition: eitcache.cpp:308
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
EITCache()
Definition: eitcache.cpp:21
static bool lock_channel(uint chanid, uint endtime)
Definition: eitcache.cpp:125
bool isActive(void) const
Definition: mythdbcon.h:204
QString GetStatistics(void) const
Definition: eitcache.cpp:50
static MSqlQueryInfo InitCon(ConnectionReuse=kNormalConnection)
Only use this in combination with MSqlQuery constructor.
Definition: mythdbcon.cpp:535
#define EITDATA
Definition: eitcache.cpp:121
uint PruneOldEntries(uint utc_timestamp)
Prunes entries that describe events ending before timestamp time.
Definition: eitcache.cpp:412
QMutex m_eventMapLock
Definition: eitcache.h:44
bool WriteChannelToDB(QStringList &value_clauses, uint chanid)
Definition: eitcache.cpp:262
static void delete_in_db(uint endtime)
Definition: eitcache.cpp:104
bool prepare(const QString &query)
QSqlQuery::prepare() is not thread safe in Qt <= 3.3.2.
Definition: mythdbcon.cpp:807
static void unlock_channel(uint chanid, uint updated)
Definition: eitcache.cpp:180
static const uint kVersionMax
Definition: eitcache.h:59
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
bool IsNewEIT(uint chanid, uint tableid, uint version, uint eventid, uint endtime)
Definition: eitcache.cpp:337
uint m_endChgCnt
Definition: eitcache.h:52
static uint extract_version(uint64_t sig)
Definition: eitcache.cpp:81
static bool modified(uint64_t sig)
Definition: eitcache.cpp:91
uint m_entryCnt
Definition: eitcache.h:53
uint m_accessCnt
Definition: eitcache.h:48
bool exec(void)
Wrap QSqlQuery::exec() so we can display SQL.
Definition: mythdbcon.cpp:603
static void DBError(const QString &where, const MSqlQuery &query)
Definition: mythdb.cpp:179
~EITCache()
Definition: eitcache.cpp:31
#define STATISTIC
Definition: eitcache.cpp:123
event_map_t * LoadChannel(uint chanid)
Definition: eitcache.cpp:217
Default UTC.
Definition: mythdate.h:14