MythTV  master
eitcache.cpp
Go to the documentation of this file.
1 // -*- Mode: c++ -*-
2 /*
3  * Copyright 2006 (C) Stuart Auchterlonie <stuarta at squashedfrog.net>
4  * Copyright 2006 (C) Janne Grunau <janne-mythtv at grunau.be>
5  * License: GPL v2
6  */
7 
8 #include <QDateTime>
9 
10 #include "libmyth/mythcontext.h"
11 #include "libmythbase/mythdate.h"
12 #include "libmythbase/mythdb.h"
14 
15 #include "eitcache.h"
16 
17 #define LOC QString("EITCache: ")
18 
19 // Highest version number. version is 5bits
20 const uint EITCache::kVersionMax = 31;
21 
23 {
24  // 24 hours ago
25  m_lastPruneTime = MythDate::current().toUTC().toSecsSinceEpoch() - 86400;
26 }
27 
29 {
30  WriteToDB();
31 }
32 
34 {
35  m_accessCnt = 0;
36  m_hitCnt = 0;
37  m_tblChgCnt = 0;
38  m_verChgCnt = 0;
39  m_endChgCnt = 0;
40  m_entryCnt = 0;
41  m_pruneCnt = 0;
42  m_prunedHitCnt = 0;
43  m_futureHitCnt = 0;
45 }
46 
47 QString EITCache::GetStatistics(void) const
48 {
49  QMutexLocker locker(&m_eventMapLock);
50  return
51  QString("Access:%1 ").arg(m_accessCnt) +
52  QString("HitRatio:%1 ").arg((m_hitCnt+m_prunedHitCnt+m_futureHitCnt+m_wrongChannelHitCnt)/(double)m_accessCnt) +
53  QString("Hits:%1 ").arg(m_hitCnt) +
54  QString("Table:%1 ").arg(m_tblChgCnt) +
55  QString("Version:%1 ").arg(m_verChgCnt) +
56  QString("Endtime:%1 ").arg(m_endChgCnt) +
57  QString("New:%1 ").arg(m_entryCnt) +
58  QString("Pruned:%1 ").arg(m_pruneCnt) +
59  QString("PrunedHits:%1 ").arg(m_prunedHitCnt) +
60  QString("Future:%1 ").arg(m_futureHitCnt) +
61  QString("WrongChannel:%1").arg(m_wrongChannelHitCnt);
62 }
63 
64 /*
65  * FIXME: This code has a builtin assumption that all timestamps will
66  * fit into a 32bit integer. Qt5.8 has switched to using a 64bit
67  * integer for timestamps.
68  */
69 static inline uint64_t construct_sig(uint tableid, uint version,
70  uint endtime, bool modified)
71 {
72  return (((uint64_t) modified << 63) | ((uint64_t) tableid << 40) |
73  ((uint64_t) version << 32) | ((uint64_t) endtime));
74 }
75 
76 static inline uint extract_table_id(uint64_t sig)
77 {
78  return (sig >> 40) & 0xff;
79 }
80 
81 static inline uint extract_version(uint64_t sig)
82 {
83  return (sig >> 32) & 0x1f;
84 }
85 
86 static inline uint extract_endtime(uint64_t sig)
87 {
88  return sig & 0xffffffff;
89 }
90 
91 static inline bool modified(uint64_t sig)
92 {
93  return (sig >> 63) != 0U;
94 }
95 
96 static void replace_in_db(QStringList &value_clauses,
97  uint chanid, uint eventid, uint64_t sig)
98 {
99  value_clauses << QString("(%1,%2,%3,%4,%5)")
100  .arg(chanid).arg(eventid).arg(extract_table_id(sig))
101  .arg(extract_version(sig)).arg(extract_endtime(sig));
102 }
103 
104 static void delete_in_db(uint endtime)
105 {
106  LOG(VB_EIT, LOG_INFO, LOC + "Deleting old cache entries from the database");
107  MSqlQuery query(MSqlQuery::InitCon());
108 
109  QString qstr =
110  "DELETE FROM eit_cache "
111  "WHERE endtime < :ENDTIME";
112 
113  query.prepare(qstr);
114  query.bindValue(":ENDTIME", endtime);
115 
116  if (!query.exec())
117  MythDB::DBError("Error deleting old eitcache entries.", query);
118 }
119 
120 
122 {
123  EITDATA = 0,
126 };
127 
128 static bool lock_channel(uint chanid, uint endtime)
129 {
130  int lock = 1;
131  MSqlQuery query(MSqlQuery::InitCon());
132 
133  QString qstr = "SELECT COUNT(*) "
134  "FROM eit_cache "
135  "WHERE chanid = :CHANID AND "
136  " endtime > :ENDTIME AND "
137  " status = :STATUS";
138 
139  query.prepare(qstr);
140  query.bindValue(":CHANID", chanid);
141  query.bindValue(":ENDTIME", endtime);
142  query.bindValue(":STATUS", CHANNEL_LOCK);
143 
144  if (!query.exec() || !query.isActive())
145  {
146  MythDB::DBError("Error checking for channel lock", query);
147  return false;
148  }
149 
150  if (query.next())
151  lock = query.value(0).toInt();
152 
153  if (lock)
154  {
155  LOG(VB_EIT, LOG_INFO,
156  LOC + QString("Ignoring channel %1 since it is locked.")
157  .arg(chanid));
158  return false;
159  }
160  uint now = MythDate::current().toSecsSinceEpoch();
161  qstr = "INSERT INTO eit_cache "
162  " ( chanid, endtime, status) "
163  "VALUES (:CHANID, :ENDTIME, :STATUS)";
164 
165  query.prepare(qstr);
166  query.bindValue(":CHANID", chanid);
167  query.bindValue(":ENDTIME", now);
168  query.bindValue(":STATUS", CHANNEL_LOCK);
169 
170  if (!query.exec())
171  {
172  MythDB::DBError("Error inserting channel lock", query);
173  return false;
174  }
175 
176  return true;
177 }
178 
179 static void unlock_channel(uint chanid, uint updated)
180 {
181  MSqlQuery query(MSqlQuery::InitCon());
182 
183  QString qstr =
184  "DELETE FROM eit_cache "
185  "WHERE chanid = :CHANID AND "
186  " status = :STATUS";
187 
188  query.prepare(qstr);
189  query.bindValue(":CHANID", chanid);
190  query.bindValue(":STATUS", CHANNEL_LOCK);
191 
192  if (!query.exec())
193  MythDB::DBError("Error deleting channel lock", query);
194 
195  // inserting statistics
196  uint now = MythDate::current().toSecsSinceEpoch();
197  qstr = "REPLACE INTO eit_cache "
198  " ( chanid, eventid, endtime, status) "
199  "VALUES (:CHANID, :EVENTID, :ENDTIME, :STATUS)";
200 
201  query.prepare(qstr);
202  query.bindValue(":CHANID", chanid);
203  query.bindValue(":EVENTID", updated);
204  query.bindValue(":ENDTIME", now);
205  query.bindValue(":STATUS", STATISTIC);
206 
207  if (!query.exec())
208  MythDB::DBError("Error inserting eit statistics", query);
209 }
210 
211 
213 {
214  if (!lock_channel(chanid, m_lastPruneTime))
215  return nullptr;
216 
217  MSqlQuery query(MSqlQuery::InitCon());
218 
219  QString qstr =
220  "SELECT eventid,tableid,version,endtime "
221  "FROM eit_cache "
222  "WHERE chanid = :CHANID AND "
223  " endtime > :ENDTIME AND "
224  " status = :STATUS";
225 
226  query.prepare(qstr);
227  query.bindValue(":CHANID", chanid);
228  query.bindValue(":ENDTIME", m_lastPruneTime);
229  query.bindValue(":STATUS", EITDATA);
230 
231  if (!query.exec() || !query.isActive())
232  {
233  MythDB::DBError("Error loading eitcache", query);
234  return nullptr;
235  }
236 
237  auto *eventMap = new event_map_t();
238 
239  while (query.next())
240  {
241  uint eventid = query.value(0).toUInt();
242  uint tableid = query.value(1).toUInt();
243  uint version = query.value(2).toUInt();
244  uint endtime = query.value(3).toUInt();
245 
246  (*eventMap)[eventid] = construct_sig(tableid, version, endtime, false);
247  }
248 
249  if (!eventMap->empty())
250  LOG(VB_EIT, LOG_INFO, LOC + QString("Loaded %1 entries for chanid %2")
251  .arg(eventMap->size()).arg(chanid));
252 
253  m_entryCnt += eventMap->size();
254  return eventMap;
255 }
256 
257 bool EITCache::WriteChannelToDB(QStringList &value_clauses, uint chanid)
258 {
259  event_map_t * eventMap = m_channelMap[chanid];
260 
261  if (!eventMap)
262  return false;
263 
264  uint size = eventMap->size();
265  uint updated = 0;
266  uint removed = 0;
267 
268  event_map_t::iterator it = eventMap->begin();
269  while (it != eventMap->end())
270  {
271  if (extract_endtime(*it) > m_lastPruneTime)
272  {
273  if (modified(*it))
274  {
275  replace_in_db(value_clauses, chanid, it.key(), *it);
276  updated++;
277  *it &= ~(uint64_t)0 >> 1; // mark as synced
278  }
279  ++it;
280  }
281  else
282  {
283  // Event is too old; remove from eit cache in memory
284  it = eventMap->erase(it);
285  removed++;
286  }
287  }
288  unlock_channel(chanid, updated);
289 
290  if (updated)
291  {
292  LOG(VB_EIT, LOG_INFO, LOC + QString("Writing %1 modified entries of %2 "
293  "for chanid %3 to database.")
294  .arg(updated).arg(size).arg(chanid));
295  }
296  if (removed)
297  {
298  LOG(VB_EIT, LOG_INFO, LOC + QString("Removed %1 old entries of %2 "
299  "for chanid %3 from cache.")
300  .arg(removed).arg(size).arg(chanid));
301  }
302  m_pruneCnt += removed;
303 
304  return true;
305 }
306 
308 {
309  QMutexLocker locker(&m_eventMapLock);
310 
311  QStringList value_clauses;
312  key_map_t::iterator it = m_channelMap.begin();
313  while (it != m_channelMap.end())
314  {
315  if (!WriteChannelToDB(value_clauses, it.key()))
316  it = m_channelMap.erase(it);
317  else
318  ++it;
319  }
320 
321  if(value_clauses.isEmpty())
322  {
323  return;
324  }
325 
326  MSqlQuery query(MSqlQuery::InitCon());
327  query.prepare(QString("REPLACE INTO eit_cache "
328  "(chanid, eventid, tableid, version, endtime) "
329  "VALUES %1").arg(value_clauses.join(",")));
330  if (!query.exec())
331  {
332  MythDB::DBError("Error updating eitcache", query);
333  }
334 }
335 
336 bool EITCache::IsNewEIT(uint chanid, uint tableid, uint version,
337  uint eventid, uint endtime)
338 {
339  m_accessCnt++;
340 
341  if ((m_accessCnt < 100000 && (m_accessCnt % 10000 == 0)) ||
342  (m_accessCnt < 1000000 && (m_accessCnt % 100000 == 0)) ||
343  (m_accessCnt % 1000000 == 0))
344  {
345  LOG(VB_EIT, LOG_INFO, LOC + GetStatistics());
346  WriteToDB();
347  }
348 
349  // don't re-add pruned entries
350  if (endtime < m_lastPruneTime)
351  {
352  m_prunedHitCnt++;
353  return false;
354  }
355 
356  // validity check, reject events with endtime over 7 weeks in the future
357  if (endtime > m_lastPruneTime + 50 * 86400)
358  {
359  m_futureHitCnt++;
360  return false;
361  }
362 
363  QMutexLocker locker(&m_eventMapLock);
364  if (!m_channelMap.contains(chanid))
365  {
366  m_channelMap[chanid] = LoadChannel(chanid);
367  }
368 
369  if (!m_channelMap[chanid])
370  {
372  return false;
373  }
374 
375  event_map_t * eventMap = m_channelMap[chanid];
376  event_map_t::iterator it = eventMap->find(eventid);
377  if (it != eventMap->end())
378  {
379  if (extract_table_id(*it) > tableid)
380  {
381  // EIT from lower (ie. better) table number
382  m_tblChgCnt++;
383  }
384  else if ((extract_table_id(*it) == tableid) &&
385  (extract_version(*it) != version))
386  {
387  // EIT updated version on current table
388  m_verChgCnt++;
389  }
390  else if (extract_endtime(*it) != endtime)
391  {
392  // Endtime (starttime + duration) changed
393  m_endChgCnt++;
394  }
395  else
396  {
397  // EIT data previously seen
398  m_hitCnt++;
399  return false;
400  }
401  }
402 
403  eventMap->insert(eventid, construct_sig(tableid, version, endtime, true));
404  m_entryCnt++;
405 
406  return true;
407 }
408 
414 {
415  if (VERBOSE_LEVEL_CHECK(VB_EIT, LOG_INFO))
416  {
417  QDateTime tmptime = MythDate::fromSecsSinceEpoch(timestamp);
418  LOG(VB_EIT, LOG_INFO,
419  LOC + "Pruning all entries that ended before UTC " +
420  tmptime.toString(Qt::ISODate));
421  }
422 
423  m_lastPruneTime = timestamp;
424 
425  // Write all modified entries to DB and start with a clean cache
426  WriteToDB();
427 
428  // Prune old entries in the DB
429  delete_in_db(timestamp);
430 
431  return 0;
432 }
433 
434 
439 {
440  MSqlQuery query(MSqlQuery::InitCon());
441 
442  QString qstr =
443  "DELETE FROM eit_cache "
444  "WHERE status = :STATUS";
445 
446  query.prepare(qstr);
447  query.bindValue(":STATUS", CHANNEL_LOCK);
448 
449  if (!query.exec())
450  MythDB::DBError("Error clearing channel locks", query);
451 }
452 
453 /* vim: set expandtab tabstop=4 shiftwidth=4: */
replace_in_db
static void replace_in_db(QStringList &value_clauses, uint chanid, uint eventid, uint64_t sig)
Definition: eitcache.cpp:96
MSqlQuery::isActive
bool isActive(void) const
Definition: mythdbcon.h:216
MSqlQuery::next
bool next(void)
Wrap QSqlQuery::next() so we can display the query results.
Definition: mythdbcon.cpp:807
MSqlQuery
QSqlQuery wrapper that fetches a DB connection from the connection pool.
Definition: mythdbcon.h:128
EITCache::m_pruneCnt
uint m_pruneCnt
Definition: eitcache.h:54
extract_endtime
static uint extract_endtime(uint64_t sig)
Definition: eitcache.cpp:86
EITCache::ClearChannelLocks
static MTV_PUBLIC void ClearChannelLocks(void)
Removes old channel locks, use it only at master backend start.
Definition: eitcache.cpp:438
LOC
#define LOC
Definition: eitcache.cpp:17
extract_table_id
static uint extract_table_id(uint64_t sig)
Definition: eitcache.cpp:76
mythdb.h
EITCache::m_verChgCnt
uint m_verChgCnt
Definition: eitcache.h:51
lock_channel
static bool lock_channel(uint chanid, uint endtime)
Definition: eitcache.cpp:128
VERBOSE_LEVEL_CHECK
static bool VERBOSE_LEVEL_CHECK(uint64_t mask, LogLevel_t level)
Definition: mythlogging.h:29
MSqlQuery::value
QVariant value(int i) const
Definition: mythdbcon.h:205
MSqlQuery::exec
bool exec(void)
Wrap QSqlQuery::exec() so we can display SQL.
Definition: mythdbcon.cpp:608
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:39
EITCache::m_channelMap
key_map_t m_channelMap
Definition: eitcache.h:42
EITCache::m_wrongChannelHitCnt
uint m_wrongChannelHitCnt
Definition: eitcache.h:57
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:14
EITCache::WriteToDB
void WriteToDB(void)
Definition: eitcache.cpp:307
EITCache::m_hitCnt
uint m_hitCnt
Definition: eitcache.h:49
EITCache::EITCache
EITCache()
Definition: eitcache.cpp:22
mythdate.h
MythDate::fromSecsSinceEpoch
MBASE_PUBLIC QDateTime fromSecsSinceEpoch(int64_t seconds)
This function takes the number of seconds since the start of the epoch and returns a QDateTime with t...
Definition: mythdate.cpp:72
EITCache::m_tblChgCnt
uint m_tblChgCnt
Definition: eitcache.h:50
mythlogging.h
unlock_channel
static void unlock_channel(uint chanid, uint updated)
Definition: eitcache.cpp:179
MSqlQuery::InitCon
static MSqlQueryInfo InitCon(ConnectionReuse _reuse=kNormalConnection)
Only use this in combination with MSqlQuery constructor.
Definition: mythdbcon.cpp:540
eitcache.h
MythDB::DBError
static void DBError(const QString &where, const MSqlQuery &query)
Definition: mythdb.cpp:227
STATISTIC
@ STATISTIC
Definition: eitcache.cpp:125
EITCache::GetStatistics
QString GetStatistics(void) const
Definition: eitcache.cpp:47
EITCache::PruneOldEntries
uint PruneOldEntries(uint utc_timestamp)
Prunes entries that describe events ending before timestamp time.
Definition: eitcache.cpp:413
uint
unsigned int uint
Definition: compat.h:79
construct_sig
static uint64_t construct_sig(uint tableid, uint version, uint endtime, bool modified)
Definition: eitcache.cpp:69
EITCache::ResetStatistics
void ResetStatistics(void)
Definition: eitcache.cpp:33
EITDATA
@ EITDATA
Definition: eitcache.cpp:123
EITCache::m_accessCnt
uint m_accessCnt
Definition: eitcache.h:48
EITCache::m_futureHitCnt
uint m_futureHitCnt
Definition: eitcache.h:56
EITCache::m_lastPruneTime
uint m_lastPruneTime
Definition: eitcache.h:45
EITCache::IsNewEIT
bool IsNewEIT(uint chanid, uint tableid, uint version, uint eventid, uint endtime)
Definition: eitcache.cpp:336
EITCache::m_eventMapLock
QMutex m_eventMapLock
Definition: eitcache.h:44
extract_version
static uint extract_version(uint64_t sig)
Definition: eitcache.cpp:81
EITCache::WriteChannelToDB
bool WriteChannelToDB(QStringList &value_clauses, uint chanid)
Definition: eitcache.cpp:257
EITCache::kVersionMax
static const uint kVersionMax
Definition: eitcache.h:59
CHANNEL_LOCK
@ CHANNEL_LOCK
Definition: eitcache.cpp:124
EITCache::m_entryCnt
uint m_entryCnt
Definition: eitcache.h:53
event_map_t
QMap< uint, uint64_t > event_map_t
Definition: eitcache.h:19
MSqlQuery::bindValue
void bindValue(const QString &placeholder, const QVariant &val)
Add a single binding.
Definition: mythdbcon.cpp:883
MythDate::ISODate
@ ISODate
Default UTC.
Definition: mythdate.h:17
EITCache::~EITCache
~EITCache()
Definition: eitcache.cpp:28
mythcontext.h
channel_status
channel_status
Definition: eitcache.cpp:121
EITCache::m_endChgCnt
uint m_endChgCnt
Definition: eitcache.h:52
EITCache::LoadChannel
event_map_t * LoadChannel(uint chanid)
Definition: eitcache.cpp:212
modified
static bool modified(uint64_t sig)
Definition: eitcache.cpp:91
delete_in_db
static void delete_in_db(uint endtime)
Definition: eitcache.cpp:104
nv_python_libs.bbciplayer.bbciplayer_api.version
string version
Definition: bbciplayer_api.py:77
EITCache::m_prunedHitCnt
uint m_prunedHitCnt
Definition: eitcache.h:55
MSqlQuery::prepare
bool prepare(const QString &query)
QSqlQuery::prepare() is not thread safe in Qt <= 3.3.2.
Definition: mythdbcon.cpp:832