MythTV  master
eitcache.cpp
Go to the documentation of this file.
1 // -*- Mode: c++ -*-
2 /*
3  * Copyright 2006 (C) Stuart Auchterlonie <stuarta at squashedfrog.net>
4  * Copyright 2006 (C) Janne Grunau <janne-mythtv at grunau.be>
5  * License: GPL v2
6  */
7 
8 #include <QDateTime>
9 
10 #include "libmythbase/mythdate.h"
11 #include "libmythbase/mythdb.h"
13 
14 #include "eitcache.h"
15 
16 #define LOC QString("EITCache: ")
17 
18 // Highest version number. version is 5bits
19 const uint EITCache::kVersionMax = 31;
20 
22 {
23  // 24 hours ago
24  m_lastPruneTime = MythDate::current().toUTC().toSecsSinceEpoch() - 86400;
25 }
26 
28 {
29  WriteToDB();
30 }
31 
33 {
34  m_accessCnt = 0;
35  m_hitCnt = 0;
36  m_tblChgCnt = 0;
37  m_verChgCnt = 0;
38  m_endChgCnt = 0;
39  m_entryCnt = 0;
40  m_pruneCnt = 0;
41  m_prunedHitCnt = 0;
42  m_futureHitCnt = 0;
44 }
45 
46 QString EITCache::GetStatistics(void) const
47 {
48  QMutexLocker locker(&m_eventMapLock);
49  return
50  QString("Access:%1 ").arg(m_accessCnt) +
51  QString("HitRatio:%1 ").arg((m_hitCnt+m_prunedHitCnt+m_futureHitCnt+m_wrongChannelHitCnt)/(double)m_accessCnt) +
52  QString("Hits:%1 ").arg(m_hitCnt) +
53  QString("Table:%1 ").arg(m_tblChgCnt) +
54  QString("Version:%1 ").arg(m_verChgCnt) +
55  QString("Endtime:%1 ").arg(m_endChgCnt) +
56  QString("New:%1 ").arg(m_entryCnt) +
57  QString("Pruned:%1 ").arg(m_pruneCnt) +
58  QString("PrunedHits:%1 ").arg(m_prunedHitCnt) +
59  QString("Future:%1 ").arg(m_futureHitCnt) +
60  QString("WrongChannel:%1").arg(m_wrongChannelHitCnt);
61 }
62 
63 /*
64  * FIXME: This code has a builtin assumption that all timestamps will
65  * fit into a 32bit integer. Qt5.8 has switched to using a 64bit
66  * integer for timestamps.
67  */
68 static inline uint64_t construct_sig(uint tableid, uint version,
69  uint endtime, bool modified)
70 {
71  return (((uint64_t) modified << 63) | ((uint64_t) tableid << 40) |
72  ((uint64_t) version << 32) | ((uint64_t) endtime));
73 }
74 
75 static inline uint extract_table_id(uint64_t sig)
76 {
77  return (sig >> 40) & 0xff;
78 }
79 
80 static inline uint extract_version(uint64_t sig)
81 {
82  return (sig >> 32) & 0x1f;
83 }
84 
85 static inline uint extract_endtime(uint64_t sig)
86 {
87  return sig & 0xffffffff;
88 }
89 
90 static inline bool modified(uint64_t sig)
91 {
92  return (sig >> 63) != 0U;
93 }
94 
95 static void replace_in_db(QStringList &value_clauses,
96  uint chanid, uint eventid, uint64_t sig)
97 {
98  value_clauses << QString("(%1,%2,%3,%4,%5)")
99  .arg(chanid).arg(eventid).arg(extract_table_id(sig))
100  .arg(extract_version(sig)).arg(extract_endtime(sig));
101 }
102 
103 static void delete_in_db(uint endtime)
104 {
105  LOG(VB_EIT, LOG_INFO, LOC + "Deleting old cache entries from the database");
106  MSqlQuery query(MSqlQuery::InitCon());
107 
108  QString qstr =
109  "DELETE FROM eit_cache "
110  "WHERE endtime < :ENDTIME";
111 
112  query.prepare(qstr);
113  query.bindValue(":ENDTIME", endtime);
114 
115  if (!query.exec())
116  MythDB::DBError("Error deleting old eitcache entries.", query);
117 }
118 
119 enum channel_status : std::uint8_t
120 {
121  EITDATA = 0,
124 };
125 
126 static bool lock_channel(uint chanid, uint endtime)
127 {
128  int lock = 1;
129  MSqlQuery query(MSqlQuery::InitCon());
130 
131  QString qstr = "SELECT COUNT(*) "
132  "FROM eit_cache "
133  "WHERE chanid = :CHANID AND "
134  " endtime > :ENDTIME AND "
135  " status = :STATUS";
136 
137  query.prepare(qstr);
138  query.bindValue(":CHANID", chanid);
139  query.bindValue(":ENDTIME", endtime);
140  query.bindValue(":STATUS", CHANNEL_LOCK);
141 
142  if (!query.exec() || !query.isActive())
143  {
144  MythDB::DBError("Error checking for channel lock", query);
145  return false;
146  }
147 
148  if (query.next())
149  lock = query.value(0).toInt();
150 
151  if (lock)
152  {
153  LOG(VB_EIT, LOG_INFO,
154  LOC + QString("Ignoring channel %1 since it is locked.")
155  .arg(chanid));
156  return false;
157  }
158  uint now = MythDate::current().toSecsSinceEpoch();
159  qstr = "INSERT INTO eit_cache "
160  " ( chanid, endtime, status) "
161  "VALUES (:CHANID, :ENDTIME, :STATUS)";
162 
163  query.prepare(qstr);
164  query.bindValue(":CHANID", chanid);
165  query.bindValue(":ENDTIME", now);
166  query.bindValue(":STATUS", CHANNEL_LOCK);
167 
168  if (!query.exec())
169  {
170  MythDB::DBError("Error inserting channel lock", query);
171  return false;
172  }
173 
174  return true;
175 }
176 
177 static void unlock_channel(uint chanid, uint updated)
178 {
179  MSqlQuery query(MSqlQuery::InitCon());
180 
181  QString qstr =
182  "DELETE FROM eit_cache "
183  "WHERE chanid = :CHANID AND "
184  " status = :STATUS";
185 
186  query.prepare(qstr);
187  query.bindValue(":CHANID", chanid);
188  query.bindValue(":STATUS", CHANNEL_LOCK);
189 
190  if (!query.exec())
191  MythDB::DBError("Error deleting channel lock", query);
192 
193  // inserting statistics
194  uint now = MythDate::current().toSecsSinceEpoch();
195  qstr = "REPLACE INTO eit_cache "
196  " ( chanid, eventid, endtime, status) "
197  "VALUES (:CHANID, :EVENTID, :ENDTIME, :STATUS)";
198 
199  query.prepare(qstr);
200  query.bindValue(":CHANID", chanid);
201  query.bindValue(":EVENTID", updated);
202  query.bindValue(":ENDTIME", now);
203  query.bindValue(":STATUS", STATISTIC);
204 
205  if (!query.exec())
206  MythDB::DBError("Error inserting eit statistics", query);
207 }
208 
210 {
211  // Event map is empty when we do not backup the cache in the database
212  if (!m_persistent)
213  {
214  return new event_map_t();
215  }
216 
217  if (!lock_channel(chanid, m_lastPruneTime))
218  return nullptr;
219 
220  MSqlQuery query(MSqlQuery::InitCon());
221 
222  QString qstr =
223  "SELECT eventid,tableid,version,endtime "
224  "FROM eit_cache "
225  "WHERE chanid = :CHANID AND "
226  " endtime > :ENDTIME AND "
227  " status = :STATUS";
228 
229  query.prepare(qstr);
230  query.bindValue(":CHANID", chanid);
231  query.bindValue(":ENDTIME", m_lastPruneTime);
232  query.bindValue(":STATUS", EITDATA);
233 
234  if (!query.exec() || !query.isActive())
235  {
236  MythDB::DBError("Error loading eitcache", query);
237  return nullptr;
238  }
239 
240  auto *eventMap = new event_map_t();
241 
242  while (query.next())
243  {
244  uint eventid = query.value(0).toUInt();
245  uint tableid = query.value(1).toUInt();
246  uint version = query.value(2).toUInt();
247  uint endtime = query.value(3).toUInt();
248 
249  (*eventMap)[eventid] = construct_sig(tableid, version, endtime, false);
250  }
251 
252  if (!eventMap->empty())
253  LOG(VB_EIT, LOG_DEBUG, LOC + QString("Loaded %1 entries for chanid %2")
254  .arg(eventMap->size()).arg(chanid));
255 
256  m_entryCnt += eventMap->size();
257  return eventMap;
258 }
259 
260 bool EITCache::WriteChannelToDB(QStringList &value_clauses, uint chanid)
261 {
262  event_map_t * eventMap = m_channelMap[chanid];
263 
264  if (!eventMap)
265  return false;
266 
267  uint size = eventMap->size();
268  uint updated = 0;
269  uint removed = 0;
270 
271  event_map_t::iterator it = eventMap->begin();
272  while (it != eventMap->end())
273  {
274  if (extract_endtime(*it) > m_lastPruneTime)
275  {
276  if (modified(*it))
277  {
278  if (m_persistent)
279  {
280  replace_in_db(value_clauses, chanid, it.key(), *it);
281  }
282 
283  updated++;
284  *it &= ~(uint64_t)0 >> 1; // Mark as synced
285  }
286  ++it;
287  }
288  else
289  {
290  // Event is too old; remove from eit cache in memory
291  it = eventMap->erase(it);
292  removed++;
293  }
294  }
295 
296  if (m_persistent)
297  {
298  unlock_channel(chanid, updated);
299  }
300 
301  if (updated)
302  {
303  if (m_persistent)
304  {
305  LOG(VB_EIT, LOG_DEBUG, LOC +
306  QString("Writing %1 modified entries of %2 for chanid %3 to database.")
307  .arg(updated).arg(size).arg(chanid));
308  }
309  else
310  {
311  LOG(VB_EIT, LOG_DEBUG, LOC +
312  QString("Updated %1 modified entries of %2 for chanid %3 in cache.")
313  .arg(updated).arg(size).arg(chanid));
314  }
315  }
316  if (removed)
317  {
318  LOG(VB_EIT, LOG_DEBUG, LOC + QString("Removed %1 old entries of %2 "
319  "for chanid %3 from cache.")
320  .arg(removed).arg(size).arg(chanid));
321  }
322  m_pruneCnt += removed;
323 
324  return true;
325 }
326 
328 {
329  QMutexLocker locker(&m_eventMapLock);
330 
331  QStringList value_clauses;
332  key_map_t::iterator it = m_channelMap.begin();
333  while (it != m_channelMap.end())
334  {
335  if (!WriteChannelToDB(value_clauses, it.key()))
336  it = m_channelMap.erase(it);
337  else
338  ++it;
339  }
340 
341  if (m_persistent)
342  {
343  if(value_clauses.isEmpty())
344  {
345  return;
346  }
347 
348  MSqlQuery query(MSqlQuery::InitCon());
349  query.prepare(QString("REPLACE INTO eit_cache "
350  "(chanid, eventid, tableid, version, endtime) "
351  "VALUES %1").arg(value_clauses.join(",")));
352  if (!query.exec())
353  {
354  MythDB::DBError("Error updating eitcache", query);
355  }
356  }
357 }
358 
359 bool EITCache::IsNewEIT(uint chanid, uint tableid, uint version,
360  uint eventid, uint endtime)
361 {
362  m_accessCnt++;
363 
364  if (m_accessCnt % 10000 == 0)
365  {
366  LOG(VB_EIT, LOG_INFO, LOC + GetStatistics());
367  ResetStatistics();
368  }
369 
370  // Don't re-add pruned entries
371  if (endtime < m_lastPruneTime)
372  {
373  m_prunedHitCnt++;
374  return false;
375  }
376 
377  // Validity check, reject events with endtime over 7 weeks in the future
378  if (endtime > m_lastPruneTime + 50 * 86400)
379  {
380  m_futureHitCnt++;
381  return false;
382  }
383 
384  QMutexLocker locker(&m_eventMapLock);
385  if (!m_channelMap.contains(chanid))
386  {
387  m_channelMap[chanid] = LoadChannel(chanid);
388  }
389 
390  if (!m_channelMap[chanid])
391  {
393  return false;
394  }
395 
396  event_map_t * eventMap = m_channelMap[chanid];
397  event_map_t::iterator it = eventMap->find(eventid);
398  if (it != eventMap->end())
399  {
400  if (extract_table_id(*it) > tableid)
401  {
402  // EIT from lower (ie. better) table number
403  m_tblChgCnt++;
404  }
405  else if ((extract_table_id(*it) == tableid) &&
406  (extract_version(*it) != version))
407  {
408  // EIT updated version on current table
409  m_verChgCnt++;
410  }
411  else if (extract_endtime(*it) != endtime)
412  {
413  // Endtime (starttime + duration) changed
414  m_endChgCnt++;
415  }
416  else
417  {
418  // EIT data previously seen
419  m_hitCnt++;
420  return false;
421  }
422  }
423 
424  eventMap->insert(eventid, construct_sig(tableid, version, endtime, true));
425  m_entryCnt++;
426 
427  return true;
428 }
429 
435 {
436  if (VERBOSE_LEVEL_CHECK(VB_EIT, LOG_DEBUG))
437  {
438  QDateTime tmptime = MythDate::fromSecsSinceEpoch(timestamp);
439  LOG(VB_EIT, LOG_INFO,
440  LOC + "Pruning all entries that ended before UTC " +
441  tmptime.toString(Qt::ISODate));
442  }
443 
444  m_lastPruneTime = timestamp;
445 
446  // Write all modified entries to DB and start with a clean cache
447  WriteToDB();
448 
449  // Prune old entries in the DB
450  if (m_persistent)
451  {
452  delete_in_db(timestamp);
453  }
454 
455  return 0;
456 }
457 
458 
463 {
464  MSqlQuery query(MSqlQuery::InitCon());
465 
466  QString qstr =
467  "DELETE FROM eit_cache "
468  "WHERE status = :STATUS";
469 
470  query.prepare(qstr);
471  query.bindValue(":STATUS", CHANNEL_LOCK);
472 
473  if (!query.exec())
474  MythDB::DBError("Error clearing channel locks", query);
475 }
476 
477 /* vim: set expandtab tabstop=4 shiftwidth=4: */
replace_in_db
static void replace_in_db(QStringList &value_clauses, uint chanid, uint eventid, uint64_t sig)
Definition: eitcache.cpp:95
MSqlQuery::isActive
bool isActive(void) const
Definition: mythdbcon.h:215
MSqlQuery::next
bool next(void)
Wrap QSqlQuery::next() so we can display the query results.
Definition: mythdbcon.cpp:812
MSqlQuery
QSqlQuery wrapper that fetches a DB connection from the connection pool.
Definition: mythdbcon.h:127
EITCache::m_pruneCnt
uint m_pruneCnt
Definition: eitcache.h:57
extract_endtime
static uint extract_endtime(uint64_t sig)
Definition: eitcache.cpp:85
EITCache::ClearChannelLocks
static MTV_PUBLIC void ClearChannelLocks(void)
Removes old channel locks, use it only at master backend start.
Definition: eitcache.cpp:462
LOC
#define LOC
Definition: eitcache.cpp:16
extract_table_id
static uint extract_table_id(uint64_t sig)
Definition: eitcache.cpp:75
mythdb.h
EITCache::m_verChgCnt
uint m_verChgCnt
Definition: eitcache.h:54
lock_channel
static bool lock_channel(uint chanid, uint endtime)
Definition: eitcache.cpp:126
channel_status
channel_status
Definition: eitcache.cpp:119
VERBOSE_LEVEL_CHECK
static bool VERBOSE_LEVEL_CHECK(uint64_t mask, LogLevel_t level)
Definition: mythlogging.h:29
MSqlQuery::value
QVariant value(int i) const
Definition: mythdbcon.h:204
MSqlQuery::exec
bool exec(void)
Wrap QSqlQuery::exec() so we can display SQL.
Definition: mythdbcon.cpp:618
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:39
EITCache::m_channelMap
key_map_t m_channelMap
Definition: eitcache.h:42
EITCache::m_wrongChannelHitCnt
uint m_wrongChannelHitCnt
Definition: eitcache.h:60
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:15
STATISTIC
@ STATISTIC
Definition: eitcache.cpp:123
EITCache::WriteToDB
void WriteToDB(void)
Definition: eitcache.cpp:327
EITCache::m_persistent
bool m_persistent
Definition: eitcache.h:48
EITCache::m_hitCnt
uint m_hitCnt
Definition: eitcache.h:52
EITCache::EITCache
EITCache()
Definition: eitcache.cpp:21
mythdate.h
MythDate::fromSecsSinceEpoch
MBASE_PUBLIC QDateTime fromSecsSinceEpoch(int64_t seconds)
This function takes the number of seconds since the start of the epoch and returns a QDateTime with t...
Definition: mythdate.cpp:81
EITCache::m_tblChgCnt
uint m_tblChgCnt
Definition: eitcache.h:53
mythlogging.h
unlock_channel
static void unlock_channel(uint chanid, uint updated)
Definition: eitcache.cpp:177
MSqlQuery::InitCon
static MSqlQueryInfo InitCon(ConnectionReuse _reuse=kNormalConnection)
Only use this in combination with MSqlQuery constructor.
Definition: mythdbcon.cpp:550
eitcache.h
MythDB::DBError
static void DBError(const QString &where, const MSqlQuery &query)
Definition: mythdb.cpp:226
EITCache::GetStatistics
QString GetStatistics(void) const
Definition: eitcache.cpp:46
EITCache::PruneOldEntries
uint PruneOldEntries(uint utc_timestamp)
Prunes entries that describe events ending before timestamp time.
Definition: eitcache.cpp:434
construct_sig
static uint64_t construct_sig(uint tableid, uint version, uint endtime, bool modified)
Definition: eitcache.cpp:68
EITCache::ResetStatistics
void ResetStatistics(void)
Definition: eitcache.cpp:32
EITCache::m_accessCnt
uint m_accessCnt
Definition: eitcache.h:51
EITCache::m_futureHitCnt
uint m_futureHitCnt
Definition: eitcache.h:59
EITCache::m_lastPruneTime
uint m_lastPruneTime
Definition: eitcache.h:45
EITCache::IsNewEIT
bool IsNewEIT(uint chanid, uint tableid, uint version, uint eventid, uint endtime)
Definition: eitcache.cpp:359
EITCache::m_eventMapLock
QMutex m_eventMapLock
Definition: eitcache.h:44
extract_version
static uint extract_version(uint64_t sig)
Definition: eitcache.cpp:80
EITCache::WriteChannelToDB
bool WriteChannelToDB(QStringList &value_clauses, uint chanid)
Definition: eitcache.cpp:260
EITCache::kVersionMax
static const uint kVersionMax
Definition: eitcache.h:62
EITCache::m_entryCnt
uint m_entryCnt
Definition: eitcache.h:56
event_map_t
QMap< uint, uint64_t > event_map_t
Definition: eitcache.h:19
MSqlQuery::bindValue
void bindValue(const QString &placeholder, const QVariant &val)
Add a single binding.
Definition: mythdbcon.cpp:888
MythDate::ISODate
@ ISODate
Default UTC.
Definition: mythdate.h:17
EITCache::~EITCache
~EITCache()
Definition: eitcache.cpp:27
EITCache::m_endChgCnt
uint m_endChgCnt
Definition: eitcache.h:55
EITCache::LoadChannel
event_map_t * LoadChannel(uint chanid)
Definition: eitcache.cpp:209
CHANNEL_LOCK
@ CHANNEL_LOCK
Definition: eitcache.cpp:122
modified
static bool modified(uint64_t sig)
Definition: eitcache.cpp:90
delete_in_db
static void delete_in_db(uint endtime)
Definition: eitcache.cpp:103
nv_python_libs.bbciplayer.bbciplayer_api.version
string version
Definition: bbciplayer_api.py:77
EITCache::m_prunedHitCnt
uint m_prunedHitCnt
Definition: eitcache.h:58
uint
unsigned int uint
Definition: freesurround.h:24
MSqlQuery::prepare
bool prepare(const QString &query)
QSqlQuery::prepare() is not thread safe in Qt <= 3.3.2.
Definition: mythdbcon.cpp:837
EITDATA
@ EITDATA
Definition: eitcache.cpp:121