MythTV  master
netgrabbermanager.cpp
Go to the documentation of this file.
1 // qt
2 #include <QCoreApplication>
3 #include <QDir>
4 #include <QFile>
5 #include <QString>
6 #include <utility>
7 
8 #include "mythdirs.h"
9 #include "mythcontext.h"
10 #include "mythsystemlegacy.h"
11 #include "exitcodes.h"
12 #include "mythdate.h"
13 #include "mythlogging.h"
14 #include "mythsystemlegacy.h"
15 
16 #include "netgrabbermanager.h"
17 #include "netutils.h"
18 
19 #define LOC QString("NetContent: ")
20 
21 // ---------------------------------------------------
22 
23 GrabberScript::GrabberScript(const QString& title, const QString& image,
24  const ArticleType &type, const QString& author,
25  const bool& search, const bool& tree,
26  const QString& description, const QString& commandline,
27  const double& version) :
28  MThread("GrabberScript")
29 {
30  m_title = title;
31  m_image = image;
32  m_type = type;
33  m_author = author;
34  m_search = search;
35  m_tree = tree;
36  m_description = description;
37  m_commandline = commandline;
39 }
40 
42 {
43  wait();
44 }
45 
47 {
48  RunProlog();
49  QMutexLocker locker(&m_lock);
50 
51  QString commandline = m_commandline;
52  MythSystemLegacy getTree(commandline, QStringList("-T"),
54  getTree.Run(900);
55  uint status = getTree.Wait();
56 
57  if( status == GENERIC_EXIT_CMD_NOT_FOUND )
58  {
59  LOG(VB_GENERAL, LOG_ERR, LOC +
60  QString("Internet Content Source %1 cannot run, file missing.")
61  .arg(m_title));
62  }
63  else if( status == GENERIC_EXIT_OK )
64  {
65  LOG(VB_GENERAL, LOG_INFO, LOC +
66  QString("Internet Content Source %1 completed download, "
67  "beginning processing...").arg(m_title));
68 
69  QByteArray result = getTree.ReadAll();
70 
71  QDomDocument domDoc;
72  domDoc.setContent(result, true);
73  QDomElement root = domDoc.documentElement();
74  QDomElement channel = root.firstChildElement("channel");
75 
77 
78  while (!channel.isNull())
79  {
80  parseDBTree(m_title, QString(), QString(), channel, GetType());
81  channel = channel.nextSiblingElement("channel");
82  }
84  LOG(VB_GENERAL, LOG_INFO, LOC +
85  QString("Internet Content Source %1 completed processing, "
86  "marking as updated.").arg(m_title));
87  }
88  else
89  {
90  LOG(VB_GENERAL, LOG_ERR, LOC +
91  QString("Internet Content Source %1 crashed while grabbing tree.")
92  .arg(m_title));
93  }
94 
95  // NOLINTNEXTLINE(readability-misleading-indentation)
96  emit finished();
97  RunEpilog();
98 }
99 
100 void GrabberScript::parseDBTree(const QString &feedtitle, const QString &path,
101  const QString &pathThumb, QDomElement& domElem,
102  const ArticleType &type)
103 {
104  QMutexLocker locker(&m_lock);
105 
106  Parse parse;
107  ResultItem::resultList articles;
108 
109  // File Handling
110  QDomElement fileitem = domElem.firstChildElement("item");
111  while (!fileitem.isNull())
112  { // Fill the article list...
113  articles.append(parse.ParseItem(fileitem));
114  fileitem = fileitem.nextSiblingElement("item");
115  }
116 
117  while (!articles.isEmpty())
118  { // Insert the articles in the DB...
119  insertTreeArticleInDB(feedtitle, path,
120  pathThumb, articles.takeFirst(), type);
121  }
122 
123  // Directory Handling
124  QDomElement diritem = domElem.firstChildElement("directory");
125  while (!diritem.isNull())
126  {
127  QDomElement subfolder = diritem;
128  QString dirname = diritem.attribute("name");
129  QString dirthumb = diritem.attribute("thumbnail");
130  dirname.replace("/", "|");
131  QString pathToUse;
132 
133  if (path.isEmpty())
134  pathToUse = dirname;
135  else
136  pathToUse = QString("%1/%2").arg(path).arg(dirname);
137 
138  parseDBTree(feedtitle,
139  pathToUse,
140  dirthumb,
141  subfolder,
142  type);
143  diritem = diritem.nextSiblingElement("directory");
144  }
145 }
146 
148 {
150  "netsite.updateFreq", 24) * 3600 * 1000);
151  m_timer = new QTimer();
152  connect( m_timer, &QTimer::timeout,
153  this, &GrabberManager::timeout);
154 }
155 
157 {
158  delete m_timer;
159 }
160 
162 {
163  m_timer->start(m_updateFreq);
164 }
165 
167 {
168  m_timer->stop();
169 }
170 
172 {
173  auto *gdt = new GrabberDownloadThread(this);
174  if (m_refreshAll)
175  gdt->refreshAll();
176  gdt->start(QThread::LowPriority);
177 
178  m_timer->start(m_updateFreq);
179 }
180 
182 {
183  QMutexLocker locker(&m_lock);
184  doUpdate();
185 }
186 
188 {
189  m_refreshAll = true;
190 }
191 
193  MThread("GrabberDownload")
194 {
195  m_parent = parent;
196 }
197 
199 {
200  cancel();
201  wait();
202 }
203 
205 {
206  m_mutex.lock();
207  qDeleteAll(m_scripts);
208  m_scripts.clear();
209  m_mutex.unlock();
210 }
211 
213 {
214  m_mutex.lock();
215  m_refreshAll = true;
216  if (!isRunning())
217  start();
218  m_mutex.unlock();
219 }
220 
222 {
223  RunProlog();
224 
226  uint updateFreq = gCoreContext->GetNumSetting(
227  "netsite.updateFreq", 24);
228 
229  while (!m_scripts.isEmpty())
230  {
231  GrabberScript *script = m_scripts.takeFirst();
232  if (script && (needsUpdate(script, updateFreq) || m_refreshAll))
233  {
234  LOG(VB_GENERAL, LOG_INFO, LOC +
235  QString("Internet Content Source %1 Updating...")
236  .arg(script->GetTitle()));
237  script->run();
238  }
239  delete script;
240  }
241  emit finished();
242  if (m_parent)
243  QCoreApplication::postEvent(m_parent, new GrabberUpdateEvent());
244 
245  RunEpilog();
246 }
247 
249 {
250  m_videoList.clear();
251 }
252 
254 {
255  resetSearch();
256 
257  delete m_searchProcess;
258  m_searchProcess = nullptr;
259 }
260 
261 
262 void Search::executeSearch(const QString &script, const QString &query,
263  const QString &pagenum)
264 {
265  resetSearch();
266 
267  LOG(VB_GENERAL, LOG_DEBUG, "Search::executeSearch");
269 
271  this, qOverload<>(&Search::slotProcessSearchExit));
273  this, qOverload<uint>(&Search::slotProcessSearchExit));
274 
275  const QString& cmd = script;
276 
277  QStringList args;
278 
279  if (!pagenum.isEmpty())
280  {
281  args.append(QString("-p"));
282  args.append(pagenum);
283  }
284 
285  args.append("-S");
286  const QString& term = query;
287  args.append(MythSystemLegacy::ShellEscape(term));
288 
289  LOG(VB_GENERAL, LOG_INFO, LOC +
290  QString("Internet Search Query: %1 %2").arg(cmd).arg(args.join(" ")));
291 
293  m_searchProcess->SetCommand(cmd, args, flags);
294  m_searchProcess->Run(40);
295 }
296 
298 {
299  qDeleteAll(m_videoList);
300  m_videoList.clear();
301 }
302 
304 {
305  Parse parse;
307 
308  QDomNodeList entries = m_document.elementsByTagName("channel");
309 
310  if (entries.count() == 0)
311  {
312  m_numResults = 0;
313  m_numReturned = 0;
314  m_numIndex = 0;
315  return;
316  }
317 
318  QDomNode itemNode = entries.item(0);
319 
320  QDomNode Node = itemNode.namedItem(QString("numresults"));
321  if (!Node.isNull())
322  {
323  m_numResults = Node.toElement().text().toUInt();
324  }
325  else
326  {
327  QDomNodeList count = m_document.elementsByTagName("item");
328 
329  if (count.count() == 0)
330  m_numResults = 0;
331  else
332  m_numResults = count.count();
333  }
334 
335  Node = itemNode.namedItem(QString("returned"));
336  if (!Node.isNull())
337  {
338  m_numReturned = Node.toElement().text().toUInt();
339  }
340  else
341  {
342  QDomNodeList items = m_document.elementsByTagName("item");
343 
344  if (items.count() == 0)
345  m_numReturned = 0;
346  else
347  m_numReturned = items.count();
348  }
349 
350  Node = itemNode.namedItem(QString("startindex"));
351  if (!Node.isNull())
352  {
353  m_numIndex = Node.toElement().text().toUInt();
354  }
355  else
356  m_numIndex = 0;
357 
358  Node = itemNode.namedItem(QString("nextpagetoken"));
359  if (!Node.isNull())
360  {
361  m_nextPageToken = Node.toElement().text();
362  }
363  else
364  m_nextPageToken = "";
365 
366  Node = itemNode.namedItem(QString("prevpagetoken"));
367  if (!Node.isNull())
368  {
369  m_prevPageToken = Node.toElement().text();
370  }
371  else
372  m_prevPageToken = "";
373 }
374 
376 {
377  if (exitcode == GENERIC_EXIT_TIMEOUT)
378  {
379  LOG(VB_GENERAL, LOG_WARNING, LOC + "Internet Search Timeout");
380 
381  if (m_searchProcess)
382  {
383  m_searchProcess->Term(true);
384  m_searchProcess->deleteLater();
385  m_searchProcess = nullptr;
386  }
387  emit searchTimedOut(this);
388  return;
389  }
390 
391  if (exitcode != GENERIC_EXIT_OK)
392  {
393  m_document.setContent(QString());
394  }
395  else
396  {
397  LOG(VB_GENERAL, LOG_INFO, LOC +
398  "Internet Search Successfully Completed");
399 
401  m_document.setContent(m_data, true);
402  }
403 
404  m_searchProcess->deleteLater();
405  m_searchProcess = nullptr;
406  emit finishedSearch(this);
407 }
408 
410 {
412 }
413 
414 void Search::SetData(QByteArray data)
415 {
416  m_data = std::move(data);
417  m_document.setContent(m_data, true);
418 
419 }
build_compdb.args
args
Definition: build_compdb.py:11
GrabberScript::GetType
const ArticleType & GetType() const
Definition: netgrabbermanager.h:36
GrabberScript::GrabberScript
GrabberScript(const QString &title, const QString &image, const ArticleType &type, const QString &author, const bool &search, const bool &tree, const QString &description, const QString &commandline, const double &version)
Definition: netgrabbermanager.cpp:23
channel
QDomElement channel
Definition: mythplugins/mytharchive/mytharchivehelper/main.cpp:501
Search::m_searchProcess
MythSystemLegacy * m_searchProcess
Definition: netgrabbermanager.h:165
Search::finishedSearch
void finishedSearch(Search *item)
MThread::start
void start(QThread::Priority p=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:288
hardwareprofile.smolt.timeout
float timeout
Definition: smolt.py:103
GrabberScript::m_type
ArticleType m_type
Definition: netgrabbermanager.h:61
GENERIC_EXIT_OK
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
insertTreeArticleInDB
bool insertTreeArticleInDB(const QString &feedtitle, const QString &path, const QString &paththumb, ResultItem *item, ArticleType type)
Definition: netutils.cpp:399
root
QDomElement root
Definition: mythplugins/mytharchive/mytharchivehelper/main.cpp:656
Search::m_numIndex
uint m_numIndex
Definition: netgrabbermanager.h:173
MythSystemLegacy::Term
void Term(bool force=false)
Definition: mythsystemlegacy.cpp:283
GrabberScript::m_lock
QMutex m_lock
Definition: netgrabbermanager.h:57
MythSystemLegacy
Definition: mythsystemlegacy.h:68
GrabberManager::m_lock
QMutex m_lock
Definition: netgrabbermanager.h:91
Search::m_numResults
uint m_numResults
Definition: netgrabbermanager.h:171
Search::~Search
~Search() override
Definition: netgrabbermanager.cpp:253
GrabberScript::m_author
QString m_author
Definition: netgrabbermanager.h:62
title
QString title
Definition: mythplugins/mytharchive/mytharchivehelper/main.cpp:636
GrabberManager::startTimer
void startTimer()
Definition: netgrabbermanager.cpp:161
GrabberScript::m_search
bool m_search
Definition: netgrabbermanager.h:63
Search::m_numReturned
uint m_numReturned
Definition: netgrabbermanager.h:172
Search::m_videoList
ResultItem::resultList m_videoList
Definition: netgrabbermanager.h:169
GrabberManager::GrabberManager
GrabberManager()
Definition: netgrabbermanager.cpp:147
arg
arg(title).arg(filename).arg(doDelete))
MythSystemLegacy::finished
void finished(void)
GrabberScript::GetTitle
const QString & GetTitle() const
Definition: netgrabbermanager.h:34
GrabberScript::m_commandline
QString m_commandline
Definition: netgrabbermanager.h:66
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23
MThread::RunProlog
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:198
MythSystemLegacy::ReadAll
QByteArray & ReadAll()
Definition: mythsystemlegacy.cpp:397
Search::m_nextPageToken
QString m_nextPageToken
Definition: netgrabbermanager.h:175
Search::SetData
void SetData(QByteArray data)
Definition: netgrabbermanager.cpp:414
GrabberDownloadThread::GrabberDownloadThread
GrabberDownloadThread(QObject *parent)
Definition: netgrabbermanager.cpp:192
GrabberDownloadThread::m_refreshAll
bool m_refreshAll
Definition: netgrabbermanager.h:133
MythSystemLegacy::Run
void Run(time_t timeout=0)
Runs a command inside the /bin/sh shell. Returns immediately.
Definition: mythsystemlegacy.cpp:212
mythdirs.h
findAllDBTreeGrabbers
GrabberScript::scriptList findAllDBTreeGrabbers()
Definition: netutils.cpp:115
needsUpdate
bool needsUpdate(GrabberScript *script, uint updateFreq)
Definition: netutils.cpp:330
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
GrabberScript::m_description
QString m_description
Definition: netgrabbermanager.h:65
markTreeUpdated
bool markTreeUpdated(GrabberScript *script, const QDateTime &curTime)
Definition: netutils.cpp:314
MythSystemLegacy::SetCommand
void SetCommand(const QString &command, uint flags)
Resets an existing MythSystemLegacy object to a new command.
Definition: mythsystemlegacy.cpp:87
mythsystemlegacy.h
Parse
Definition: rssparse.h:189
mythdate.h
GrabberDownloadThread::m_parent
QObject * m_parent
Definition: netgrabbermanager.h:130
mythlogging.h
Search::m_document
QDomDocument m_document
Definition: netgrabbermanager.h:168
Parse::ParseItem
ResultItem * ParseItem(const QDomElement &item) const
Definition: rssparse.cpp:733
Search::process
void process(void)
Definition: netgrabbermanager.cpp:303
netgrabbermanager.h
MythSystemLegacy::error
void error(uint status)
Search::m_prevPageToken
QString m_prevPageToken
Definition: netgrabbermanager.h:176
MythSystemLegacy::ShellEscape
static QString ShellEscape(const QString &in)
Definition: mythsystemlegacy.cpp:477
netutils.h
GENERIC_EXIT_CMD_NOT_FOUND
#define GENERIC_EXIT_CMD_NOT_FOUND
Command not found.
Definition: exitcodes.h:12
MThread::RunEpilog
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:211
GrabberScript
Definition: netgrabbermanager.h:17
GrabberUpdateEvent
Definition: netgrabbermanager.h:102
GrabberScript::run
void run(void) override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
Definition: netgrabbermanager.cpp:46
GrabberScript::m_title
QString m_title
Definition: netgrabbermanager.h:59
GENERIC_EXIT_TIMEOUT
#define GENERIC_EXIT_TIMEOUT
Process timed out.
Definition: exitcodes.h:24
uint
unsigned int uint
Definition: compat.h:140
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:56
GrabberScript::m_image
QString m_image
Definition: netgrabbermanager.h:60
MythCoreContext::GetNumSetting
int GetNumSetting(const QString &key, int defaultval=0)
Definition: mythcorecontext.cpp:929
GrabberScript::m_version
double m_version
Definition: netgrabbermanager.h:67
kMSRunShell
@ kMSRunShell
run process through shell
Definition: mythsystem.h:41
GrabberScript::m_tree
bool m_tree
Definition: netgrabbermanager.h:64
ArticleType
ArticleType
Definition: rssparse.h:20
kMSRunBackground
@ kMSRunBackground
run child in the background
Definition: mythsystem.h:36
GrabberDownloadThread::refreshAll
void refreshAll()
Definition: netgrabbermanager.cpp:212
GrabberManager::m_refreshAll
bool m_refreshAll
Definition: netgrabbermanager.h:96
GrabberManager::m_timer
QTimer * m_timer
Definition: netgrabbermanager.h:92
GrabberManager::~GrabberManager
~GrabberManager() override
Definition: netgrabbermanager.cpp:156
gdt
GrabberDownloadThread * gdt
Definition: mythplugins/mythnetvision/mythfillnetvision/main.cpp:26
GrabberManager::doUpdate
void doUpdate()
Definition: netgrabbermanager.cpp:171
GrabberScript::~GrabberScript
~GrabberScript() override
Definition: netgrabbermanager.cpp:41
clearTreeItems
bool clearTreeItems(const QString &feedcommand)
Definition: netutils.cpp:360
GrabberDownloadThread::run
void run() override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
Definition: netgrabbermanager.cpp:221
Search::Search
Search()
Definition: netgrabbermanager.cpp:248
Search::m_data
QByteArray m_data
Definition: netgrabbermanager.h:167
Search::searchTimedOut
void searchTimedOut(Search *item)
GrabberDownloadThread::m_mutex
QMutex m_mutex
Definition: netgrabbermanager.h:132
GrabberDownloadThread::~GrabberDownloadThread
~GrabberDownloadThread() override
Definition: netgrabbermanager.cpp:198
Search::resetSearch
void resetSearch(void)
Definition: netgrabbermanager.cpp:297
GrabberScript::finished
void finished(void)
Parse::parseRSS
ResultItem::resultList parseRSS(const QDomDocument &domDoc) const
Definition: rssparse.cpp:710
Search::slotProcessSearchExit
void slotProcessSearchExit(void)
Definition: netgrabbermanager.cpp:409
GrabberManager::timeout
void timeout(void)
Definition: netgrabbermanager.cpp:181
GrabberManager::stopTimer
void stopTimer()
Definition: netgrabbermanager.cpp:166
mythcontext.h
MThread
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:49
MThread::isRunning
bool isRunning(void) const
Definition: mthread.cpp:268
ResultItem::resultList
QList< ResultItem * > resultList
Definition: rssparse.h:114
GrabberDownloadThread::m_scripts
QList< GrabberScript * > m_scripts
Definition: netgrabbermanager.h:131
GrabberDownloadThread::cancel
void cancel()
Definition: netgrabbermanager.cpp:204
GrabberManager::refreshAll
void refreshAll()
Definition: netgrabbermanager.cpp:187
MThread::wait
bool wait(unsigned long time=ULONG_MAX)
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:305
exitcodes.h
Search::executeSearch
void executeSearch(const QString &script, const QString &query, const QString &pagenum="")
Definition: netgrabbermanager.cpp:262
GrabberManager::m_updateFreq
uint m_updateFreq
Definition: netgrabbermanager.h:94
query
MSqlQuery query(MSqlQuery::InitCon())
kMSStdOut
@ kMSStdOut
allow access to stdout
Definition: mythsystem.h:39
nv_python_libs.bbciplayer.bbciplayer_api.version
string version
Definition: bbciplayer_api.py:81
GrabberDownloadThread::finished
void finished()
GrabberScript::parseDBTree
void parseDBTree(const QString &feedtitle, const QString &path, const QString &pathThumb, QDomElement &domElem, const ArticleType &type)
Definition: netgrabbermanager.cpp:100
GrabberDownloadThread
Definition: netgrabbermanager.h:110
MythSystemLegacy::Wait
uint Wait(time_t timeout=0)
Definition: mythsystemlegacy.cpp:242
LOC
#define LOC
Definition: netgrabbermanager.cpp:19