MythTV  master
eitcache.cpp
Go to the documentation of this file.
1 // -*- Mode: c++ -*-
2 /*
3  * Copyright 2006 (C) Stuart Auchterlonie <stuarta at squashedfrog.net>
4  * Copyright 2006 (C) Janne Grunau <janne-mythtv at grunau.be>
5  * License: GPL v2
6  */
7 
8 #include <QDateTime>
9 
10 #include "eitcache.h"
11 #include "mythcontext.h"
12 #include "mythdb.h"
13 #include "mythlogging.h"
14 #include "mythdate.h"
15 
16 #define LOC QString("EITCache: ")
17 
18 // Highest version number. version is 5bits
19 const uint EITCache::kVersionMax = 31;
20 
22 {
23  // 24 hours ago
24  m_lastPruneTime = MythDate::current().toUTC().toSecsSinceEpoch() - 86400;
25 }
26 
28 {
29  WriteToDB();
30 }
31 
33 {
34  m_accessCnt = 0;
35  m_hitCnt = 0;
36  m_tblChgCnt = 0;
37  m_verChgCnt = 0;
38  m_endChgCnt = 0;
39  m_entryCnt = 0;
40  m_pruneCnt = 0;
41  m_prunedHitCnt = 0;
42  m_futureHitCnt = 0;
44 }
45 
46 QString EITCache::GetStatistics(void) const
47 {
48  QMutexLocker locker(&m_eventMapLock);
49  return QString(
50  "EITCache stats: Access:%1 Hits:%2 "
51  "Table:%3 Version:%4 Endtime:%5 New:%6 "
52  "Pruned:%7 Pruned Hits:%8 Future:%9 Wrong Channel:%10 "
53  "Hit Ratio:%11")
54  .arg(m_accessCnt).arg(m_hitCnt)
55  .arg(m_tblChgCnt).arg(m_verChgCnt).arg(m_endChgCnt).arg(m_entryCnt)
58 }
59 
60 /*
61  * FIXME: This code has a builtin assumption that all timestamps will
62  * fit into a 32bit integer. Qt5.8 has switched to using a 64bit
63  * integer for timestamps.
64  */
65 static inline uint64_t construct_sig(uint tableid, uint version,
66  uint endtime, bool modified)
67 {
68  return (((uint64_t) modified << 63) | ((uint64_t) tableid << 40) |
69  ((uint64_t) version << 32) | ((uint64_t) endtime));
70 }
71 
72 static inline uint extract_table_id(uint64_t sig)
73 {
74  return (sig >> 40) & 0xff;
75 }
76 
77 static inline uint extract_version(uint64_t sig)
78 {
79  return (sig >> 32) & 0x1f;
80 }
81 
82 static inline uint extract_endtime(uint64_t sig)
83 {
84  return sig & 0xffffffff;
85 }
86 
87 static inline bool modified(uint64_t sig)
88 {
89  return (sig >> 63) != 0U;
90 }
91 
92 static void replace_in_db(QStringList &value_clauses,
93  uint chanid, uint eventid, uint64_t sig)
94 {
95  value_clauses << QString("(%1,%2,%3,%4,%5)")
96  .arg(chanid).arg(eventid).arg(extract_table_id(sig))
97  .arg(extract_version(sig)).arg(extract_endtime(sig));
98 }
99 
100 static void delete_in_db(uint endtime)
101 {
102  LOG(VB_EIT, LOG_INFO, LOC + "Deleting old cache entries from the database");
104 
105  QString qstr =
106  "DELETE FROM eit_cache "
107  "WHERE endtime < :ENDTIME";
108 
109  query.prepare(qstr);
110  query.bindValue(":ENDTIME", endtime);
111 
112  if (!query.exec())
113  MythDB::DBError("Error deleting old eitcache entries.", query);
114 }
115 
116 
117 #define EITDATA 0
118 #define CHANNEL_LOCK 1
119 #define STATISTIC 2
120 
121 static bool lock_channel(uint chanid, uint endtime)
122 {
123  int lock = 1;
125 
126  QString qstr = "SELECT COUNT(*) "
127  "FROM eit_cache "
128  "WHERE chanid = :CHANID AND "
129  " endtime > :ENDTIME AND "
130  " status = :STATUS";
131 
132  query.prepare(qstr);
133  query.bindValue(":CHANID", chanid);
134  query.bindValue(":ENDTIME", endtime);
135  query.bindValue(":STATUS", CHANNEL_LOCK);
136 
137  if (!query.exec() || !query.isActive())
138  {
139  MythDB::DBError("Error checking for channel lock", query);
140  return false;
141  }
142 
143  if (query.next())
144  lock = query.value(0).toInt();
145 
146  if (lock)
147  {
148  LOG(VB_EIT, LOG_INFO,
149  LOC + QString("Ignoring channel %1 since it is locked.")
150  .arg(chanid));
151  return false;
152  }
153  uint now = MythDate::current().toSecsSinceEpoch();
154  qstr = "INSERT INTO eit_cache "
155  " ( chanid, endtime, status) "
156  "VALUES (:CHANID, :ENDTIME, :STATUS)";
157 
158  query.prepare(qstr);
159  query.bindValue(":CHANID", chanid);
160  query.bindValue(":ENDTIME", now);
161  query.bindValue(":STATUS", CHANNEL_LOCK);
162 
163  if (!query.exec())
164  {
165  MythDB::DBError("Error inserting channel lock", query);
166  return false;
167  }
168 
169  return true;
170 }
171 
172 static void unlock_channel(uint chanid, uint updated)
173 {
175 
176  QString qstr =
177  "DELETE FROM eit_cache "
178  "WHERE chanid = :CHANID AND "
179  " status = :STATUS";
180 
181  query.prepare(qstr);
182  query.bindValue(":CHANID", chanid);
183  query.bindValue(":STATUS", CHANNEL_LOCK);
184 
185  if (!query.exec())
186  MythDB::DBError("Error deleting channel lock", query);
187 
188  // inserting statistics
189  uint now = MythDate::current().toSecsSinceEpoch();
190  qstr = "REPLACE INTO eit_cache "
191  " ( chanid, eventid, endtime, status) "
192  "VALUES (:CHANID, :EVENTID, :ENDTIME, :STATUS)";
193 
194  query.prepare(qstr);
195  query.bindValue(":CHANID", chanid);
196  query.bindValue(":EVENTID", updated);
197  query.bindValue(":ENDTIME", now);
198  query.bindValue(":STATUS", STATISTIC);
199 
200  if (!query.exec())
201  MythDB::DBError("Error inserting eit statistics", query);
202 }
203 
204 
206 {
207  if (!lock_channel(chanid, m_lastPruneTime))
208  return nullptr;
209 
211 
212  QString qstr =
213  "SELECT eventid,tableid,version,endtime "
214  "FROM eit_cache "
215  "WHERE chanid = :CHANID AND "
216  " endtime > :ENDTIME AND "
217  " status = :STATUS";
218 
219  query.prepare(qstr);
220  query.bindValue(":CHANID", chanid);
221  query.bindValue(":ENDTIME", m_lastPruneTime);
222  query.bindValue(":STATUS", EITDATA);
223 
224  if (!query.exec() || !query.isActive())
225  {
226  MythDB::DBError("Error loading eitcache", query);
227  return nullptr;
228  }
229 
230  auto *eventMap = new event_map_t();
231 
232  while (query.next())
233  {
234  uint eventid = query.value(0).toUInt();
235  uint tableid = query.value(1).toUInt();
236  uint version = query.value(2).toUInt();
237  uint endtime = query.value(3).toUInt();
238 
239  (*eventMap)[eventid] = construct_sig(tableid, version, endtime, false);
240  }
241 
242  if (!eventMap->empty())
243  LOG(VB_EIT, LOG_INFO, LOC + QString("Loaded %1 entries for channel %2")
244  .arg(eventMap->size()).arg(chanid));
245 
246  m_entryCnt += eventMap->size();
247  return eventMap;
248 }
249 
250 bool EITCache::WriteChannelToDB(QStringList &value_clauses, uint chanid)
251 {
252  event_map_t * eventMap = m_channelMap[chanid];
253 
254  if (!eventMap)
255  return false;
256 
257  uint size = eventMap->size();
258  uint updated = 0;
259  uint removed = 0;
260 
261  event_map_t::iterator it = eventMap->begin();
262  while (it != eventMap->end())
263  {
264  if (extract_endtime(*it) > m_lastPruneTime)
265  {
266  if (modified(*it))
267  {
268  replace_in_db(value_clauses, chanid, it.key(), *it);
269  updated++;
270  *it &= ~(uint64_t)0 >> 1; // mark as synced
271  }
272  ++it;
273  }
274  else
275  {
276  // Event is too old; remove from eit cache in memory
277  it = eventMap->erase(it);
278  removed++;
279  }
280  }
281  unlock_channel(chanid, updated);
282 
283  if (updated)
284  {
285  LOG(VB_EIT, LOG_INFO, LOC + QString("Writing %1 modified entries of %2 "
286  "for channel %3 to database.")
287  .arg(updated).arg(size).arg(chanid));
288  }
289  if (removed)
290  {
291  LOG(VB_EIT, LOG_INFO, LOC + QString("Removed %1 old entries of %2 "
292  "for channel %3 from cache.")
293  .arg(removed).arg(size).arg(chanid));
294  }
295  m_pruneCnt += removed;
296 
297  return true;
298 }
299 
301 {
302  QMutexLocker locker(&m_eventMapLock);
303 
304  QStringList value_clauses;
305  key_map_t::iterator it = m_channelMap.begin();
306  while (it != m_channelMap.end())
307  {
308  if (!WriteChannelToDB(value_clauses, it.key()))
309  it = m_channelMap.erase(it);
310  else
311  ++it;
312  }
313 
314  if(value_clauses.isEmpty())
315  {
316  return;
317  }
318 
320  query.prepare(QString("REPLACE INTO eit_cache "
321  "(chanid, eventid, tableid, version, endtime) "
322  "VALUES %1").arg(value_clauses.join(",")));
323  if (!query.exec())
324  {
325  MythDB::DBError("Error updating eitcache", query);
326  }
327 }
328 
329 bool EITCache::IsNewEIT(uint chanid, uint tableid, uint version,
330  uint eventid, uint endtime)
331 {
332  m_accessCnt++;
333 
334  if (m_accessCnt % 500000 == 50000)
335  {
336  LOG(VB_EIT, LOG_INFO, GetStatistics());
337  WriteToDB();
338  }
339 
340  // don't re-add pruned entries
341  if (endtime < m_lastPruneTime)
342  {
343  m_prunedHitCnt++;
344  return false;
345  }
346 
347  // validity check, reject events with endtime over 7 weeks in the future
348  if (endtime > m_lastPruneTime + 50 * 86400)
349  {
350  m_futureHitCnt++;
351  return false;
352  }
353 
354  QMutexLocker locker(&m_eventMapLock);
355  if (!m_channelMap.contains(chanid))
356  {
357  m_channelMap[chanid] = LoadChannel(chanid);
358  }
359 
360  if (!m_channelMap[chanid])
361  {
363  return false;
364  }
365 
366  event_map_t * eventMap = m_channelMap[chanid];
367  event_map_t::iterator it = eventMap->find(eventid);
368  if (it != eventMap->end())
369  {
370  if (extract_table_id(*it) > tableid)
371  {
372  // EIT from lower (ie. better) table number
373  m_tblChgCnt++;
374  }
375  else if ((extract_table_id(*it) == tableid) &&
376  (extract_version(*it) != version))
377  {
378  // EIT updated version on current table
379  m_verChgCnt++;
380  }
381  else if (extract_endtime(*it) != endtime)
382  {
383  // Endtime (starttime + duration) changed
384  m_endChgCnt++;
385  }
386  else
387  {
388  // EIT data previously seen
389  m_hitCnt++;
390  return false;
391  }
392  }
393 
394  eventMap->insert(eventid, construct_sig(tableid, version, endtime, true));
395  m_entryCnt++;
396 
397  return true;
398 }
399 
405 {
406  if (VERBOSE_LEVEL_CHECK(VB_EIT, LOG_INFO))
407  {
408  QDateTime tmptime = MythDate::fromSecsSinceEpoch(timestamp);
409  LOG(VB_EIT, LOG_INFO,
410  LOC + "Pruning all entries that ended before UTC " +
411  tmptime.toString(Qt::ISODate));
412  }
413 
414  m_lastPruneTime = timestamp;
415 
416  // Write all modified entries to DB and start with a clean cache
417  WriteToDB();
418 
419  // Prune old entries in the DB
420  delete_in_db(timestamp);
421 
422  return 0;
423 }
424 
425 
430 {
432 
433  QString qstr =
434  "DELETE FROM eit_cache "
435  "WHERE status = :STATUS";
436 
437  query.prepare(qstr);
438  query.bindValue(":STATUS", CHANNEL_LOCK);
439 
440  if (!query.exec())
441  MythDB::DBError("Error clearing channel locks", query);
442 }
443 
444 /* vim: set expandtab tabstop=4 shiftwidth=4: */
replace_in_db
static void replace_in_db(QStringList &value_clauses, uint chanid, uint eventid, uint64_t sig)
Definition: eitcache.cpp:92
MSqlQuery::isActive
bool isActive(void) const
Definition: mythdbcon.h:204
MSqlQuery::next
bool next(void)
Wrap QSqlQuery::next() so we can display the query results.
Definition: mythdbcon.cpp:783
MSqlQuery
QSqlQuery wrapper that fetches a DB connection from the connection pool.
Definition: mythdbcon.h:126
EITCache::m_pruneCnt
uint m_pruneCnt
Definition: eitcache.h:54
extract_endtime
static uint extract_endtime(uint64_t sig)
Definition: eitcache.cpp:82
EITCache::ClearChannelLocks
static MTV_PUBLIC void ClearChannelLocks(void)
removes old channel locks, use it only at master backend start
Definition: eitcache.cpp:429
LOC
#define LOC
Definition: eitcache.cpp:16
extract_table_id
static uint extract_table_id(uint64_t sig)
Definition: eitcache.cpp:72
mythdb.h
EITCache::m_verChgCnt
uint m_verChgCnt
Definition: eitcache.h:51
lock_channel
static bool lock_channel(uint chanid, uint endtime)
Definition: eitcache.cpp:121
MSqlQuery::value
QVariant value(int i) const
Definition: mythdbcon.h:198
arg
arg(title).arg(filename).arg(doDelete))
MSqlQuery::exec
bool exec(void)
Wrap QSqlQuery::exec() so we can display SQL.
Definition: mythdbcon.cpp:603
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23
EITCache::m_channelMap
key_map_t m_channelMap
Definition: eitcache.h:42
EITCache::m_wrongChannelHitCnt
uint m_wrongChannelHitCnt
Definition: eitcache.h:57
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
CHANNEL_LOCK
#define CHANNEL_LOCK
Definition: eitcache.cpp:118
EITCache::WriteToDB
void WriteToDB(void)
Definition: eitcache.cpp:300
EITCache::m_hitCnt
uint m_hitCnt
Definition: eitcache.h:49
MythDate::fromSecsSinceEpoch
MBASE_PUBLIC QDateTime fromSecsSinceEpoch(uint seconds)
This function takes the number of seconds since the start of the epoch and returns a QDateTime with t...
Definition: mythdate.cpp:68
EITCache::EITCache
EITCache()
Definition: eitcache.cpp:21
mythdate.h
EITCache::m_tblChgCnt
uint m_tblChgCnt
Definition: eitcache.h:50
mythlogging.h
unlock_channel
static void unlock_channel(uint chanid, uint updated)
Definition: eitcache.cpp:172
MSqlQuery::InitCon
static MSqlQueryInfo InitCon(ConnectionReuse _reuse=kNormalConnection)
Only use this in combination with MSqlQuery constructor.
Definition: mythdbcon.cpp:535
eitcache.h
MythDB::DBError
static void DBError(const QString &where, const MSqlQuery &query)
Definition: mythdb.cpp:179
EITDATA
#define EITDATA
Definition: eitcache.cpp:117
EITCache::GetStatistics
QString GetStatistics(void) const
Definition: eitcache.cpp:46
EITCache::PruneOldEntries
uint PruneOldEntries(uint utc_timestamp)
Prunes entries that describe events ending before timestamp time.
Definition: eitcache.cpp:404
uint
unsigned int uint
Definition: compat.h:140
construct_sig
static uint64_t construct_sig(uint tableid, uint version, uint endtime, bool modified)
Definition: eitcache.cpp:65
EITCache::ResetStatistics
void ResetStatistics(void)
Definition: eitcache.cpp:32
EITCache::m_accessCnt
uint m_accessCnt
Definition: eitcache.h:48
EITCache::m_futureHitCnt
uint m_futureHitCnt
Definition: eitcache.h:56
EITCache::m_lastPruneTime
uint m_lastPruneTime
Definition: eitcache.h:45
EITCache::IsNewEIT
bool IsNewEIT(uint chanid, uint tableid, uint version, uint eventid, uint endtime)
Definition: eitcache.cpp:329
EITCache::m_eventMapLock
QMutex m_eventMapLock
Definition: eitcache.h:44
extract_version
static uint extract_version(uint64_t sig)
Definition: eitcache.cpp:77
EITCache::WriteChannelToDB
bool WriteChannelToDB(QStringList &value_clauses, uint chanid)
Definition: eitcache.cpp:250
VERBOSE_LEVEL_CHECK
#define VERBOSE_LEVEL_CHECK(_MASK_, _LEVEL_)
Definition: mythlogging.h:14
EITCache::kVersionMax
static const uint kVersionMax
Definition: eitcache.h:59
EITCache::m_entryCnt
uint m_entryCnt
Definition: eitcache.h:53
event_map_t
QMap< uint, uint64_t > event_map_t
Definition: eitcache.h:19
MSqlQuery::bindValue
void bindValue(const QString &placeholder, const QVariant &val)
Add a single binding.
Definition: mythdbcon.cpp:864
MythDate::ISODate
@ ISODate
Default UTC.
Definition: mythdate.h:14
EITCache::~EITCache
~EITCache()
Definition: eitcache.cpp:27
mythcontext.h
EITCache::m_endChgCnt
uint m_endChgCnt
Definition: eitcache.h:52
EITCache::LoadChannel
event_map_t * LoadChannel(uint chanid)
Definition: eitcache.cpp:205
STATISTIC
#define STATISTIC
Definition: eitcache.cpp:119
modified
static bool modified(uint64_t sig)
Definition: eitcache.cpp:87
delete_in_db
static void delete_in_db(uint endtime)
Definition: eitcache.cpp:100
query
MSqlQuery query(MSqlQuery::InitCon())
nv_python_libs.bbciplayer.bbciplayer_api.version
string version
Definition: bbciplayer_api.py:81
EITCache::m_prunedHitCnt
uint m_prunedHitCnt
Definition: eitcache.h:55
MSqlQuery::prepare
bool prepare(const QString &query)
QSqlQuery::prepare() is not thread safe in Qt <= 3.3.2.
Definition: mythdbcon.cpp:808