MythTV  master
netgrabbermanager.cpp
Go to the documentation of this file.
1 // qt
2 #include <QCoreApplication>
3 #include <QDir>
4 #include <QFile>
5 #include <QString>
6 #include <utility>
7 
9 #include "libmythbase/mythdate.h"
10 #include "libmythbase/mythdirs.h"
13 
14 #include "mythcontext.h"
15 #include "netgrabbermanager.h"
16 #include "netutils.h"
17 
18 #define LOC QString("NetContent: ")
19 
20 // ---------------------------------------------------
21 
22 GrabberScript::GrabberScript(const QString& title, const QString& image,
23  const ArticleType type, const QString& author,
24  const bool search, const bool tree,
25  const QString& description, const QString& commandline,
26  const double version) :
27  MThread("GrabberScript")
28 {
29  m_title = title;
30  m_image = image;
31  m_type = type;
32  m_author = author;
33  m_search = search;
34  m_tree = tree;
35  m_description = description;
36  m_commandline = commandline;
38 }
39 
41 {
42  wait();
43 }
44 
46 {
47  RunProlog();
48  QMutexLocker locker(&m_lock);
49 
50  QString commandline = m_commandline;
51  MythSystemLegacy getTree(commandline, QStringList("-T"),
53  getTree.Run(15min);
54  uint status = getTree.Wait();
55 
56  if( status == GENERIC_EXIT_CMD_NOT_FOUND )
57  {
58  LOG(VB_GENERAL, LOG_ERR, LOC +
59  QString("Internet Content Source %1 cannot run, file missing.")
60  .arg(m_title));
61  }
62  else if( status == GENERIC_EXIT_OK )
63  {
64  LOG(VB_GENERAL, LOG_INFO, LOC +
65  QString("Internet Content Source %1 completed download, "
66  "beginning processing...").arg(m_title));
67 
68  QByteArray result = getTree.ReadAll();
69 
70  QDomDocument domDoc;
71  domDoc.setContent(result, true);
72  QDomElement root = domDoc.documentElement();
73  QDomElement channel = root.firstChildElement("channel");
74 
76 
77  while (!channel.isNull())
78  {
79  parseDBTree(m_title, QString(), QString(), channel, GetType());
80  channel = channel.nextSiblingElement("channel");
81  }
83  LOG(VB_GENERAL, LOG_INFO, LOC +
84  QString("Internet Content Source %1 completed processing, "
85  "marking as updated.").arg(m_title));
86  }
87  else
88  {
89  LOG(VB_GENERAL, LOG_ERR, LOC +
90  QString("Internet Content Source %1 crashed while grabbing tree.")
91  .arg(m_title));
92  }
93 
94  // NOLINTNEXTLINE(readability-misleading-indentation)
95  emit finished();
96  RunEpilog();
97 }
98 
99 void GrabberScript::parseDBTree(const QString &feedtitle, const QString &path,
100  const QString &pathThumb, QDomElement& domElem,
101  const ArticleType type)
102 {
103  QMutexLocker locker(&m_lock);
104 
105  Parse parse;
106  ResultItem::resultList articles;
107 
108  // File Handling
109  QDomElement fileitem = domElem.firstChildElement("item");
110  while (!fileitem.isNull())
111  { // Fill the article list...
112  articles.append(Parse::ParseItem(fileitem));
113  fileitem = fileitem.nextSiblingElement("item");
114  }
115 
116  while (!articles.isEmpty())
117  { // Insert the articles in the DB...
118  insertTreeArticleInDB(feedtitle, path,
119  pathThumb, articles.takeFirst(), type);
120  }
121 
122  // Directory Handling
123  QDomElement diritem = domElem.firstChildElement("directory");
124  while (!diritem.isNull())
125  {
126  QDomElement subfolder = diritem;
127  QString dirname = diritem.attribute("name");
128  QString dirthumb = diritem.attribute("thumbnail");
129  dirname.replace("/", "|");
130  QString pathToUse;
131 
132  if (path.isEmpty())
133  pathToUse = dirname;
134  else
135  pathToUse = QString("%1/%2").arg(path, dirname);
136 
137  parseDBTree(feedtitle,
138  pathToUse,
139  dirthumb,
140  subfolder,
141  type);
142  diritem = diritem.nextSiblingElement("directory");
143  }
144 }
145 
147 {
148  m_updateFreq = gCoreContext->GetDurSetting<std::chrono::hours>("netsite.updateFreq", 24h);
149  m_timer = new QTimer();
150  connect( m_timer, &QTimer::timeout,
151  this, &GrabberManager::timeout);
152 }
153 
155 {
156  delete m_timer;
157 }
158 
160 {
161  m_timer->start(m_updateFreq);
162 }
163 
165 {
166  m_timer->stop();
167 }
168 
170 {
171  auto *gdt = new GrabberDownloadThread(this);
172  if (m_refreshAll)
173  gdt->refreshAll();
174  gdt->start(QThread::LowPriority);
175 
176  m_timer->start(m_updateFreq);
177 }
178 
180 {
181  QMutexLocker locker(&m_lock);
182  doUpdate();
183 }
184 
186 {
187  m_refreshAll = true;
188 }
189 
191  MThread("GrabberDownload")
192 {
193  m_parent = parent;
194 }
195 
197 {
198  cancel();
199  wait();
200 }
201 
203 {
204  m_mutex.lock();
205  qDeleteAll(m_scripts);
206  m_scripts.clear();
207  m_mutex.unlock();
208 }
209 
211 {
212  m_mutex.lock();
213  m_refreshAll = true;
214  if (!isRunning())
215  start();
216  m_mutex.unlock();
217 }
218 
220 {
221  RunProlog();
222 
224  auto updateFreq = gCoreContext->GetDurSetting<std::chrono::hours>(
225  "netsite.updateFreq", 24h);
226 
227  while (!m_scripts.isEmpty())
228  {
229  GrabberScript *script = m_scripts.takeFirst();
230  if (script && (needsUpdate(script, updateFreq) || m_refreshAll))
231  {
232  LOG(VB_GENERAL, LOG_INFO, LOC +
233  QString("Internet Content Source %1 Updating...")
234  .arg(script->GetTitle()));
235  script->run();
236  }
237  delete script;
238  }
239  emit finished();
240  if (m_parent)
241  QCoreApplication::postEvent(m_parent, new GrabberUpdateEvent());
242 
243  RunEpilog();
244 }
245 
247 {
248  m_videoList.clear();
249 }
250 
252 {
253  resetSearch();
254 
255  delete m_searchProcess;
256  m_searchProcess = nullptr;
257 }
258 
259 
260 void Search::executeSearch(const QString &script, const QString &query,
261  const QString &pagenum)
262 {
263  resetSearch();
264 
265  LOG(VB_GENERAL, LOG_DEBUG, "Search::executeSearch");
267 
269  this, qOverload<>(&Search::slotProcessSearchExit));
271  this, qOverload<uint>(&Search::slotProcessSearchExit));
272 
273  const QString& cmd = script;
274 
275  QStringList args;
276 
277  if (!pagenum.isEmpty())
278  {
279  args.append(QString("-p"));
280  args.append(pagenum);
281  }
282 
283  args.append("-S");
284  const QString& term = query;
285  args.append(MythSystemLegacy::ShellEscape(term));
286 
287  LOG(VB_GENERAL, LOG_INFO, LOC +
288  QString("Internet Search Query: %1 %2").arg(cmd, args.join(" ")));
289 
291  m_searchProcess->SetCommand(cmd, args, flags);
292  m_searchProcess->Run(40s);
293 }
294 
296 {
297  qDeleteAll(m_videoList);
298  m_videoList.clear();
299 }
300 
302 {
303  Parse parse;
305 
306  QDomNodeList entries = m_document.elementsByTagName("channel");
307 
308  if (entries.count() == 0)
309  {
310  m_numResults = 0;
311  m_numReturned = 0;
312  m_numIndex = 0;
313  return;
314  }
315 
316  QDomNode itemNode = entries.item(0);
317 
318  QDomNode Node = itemNode.namedItem(QString("numresults"));
319  if (!Node.isNull())
320  {
321  m_numResults = Node.toElement().text().toUInt();
322  }
323  else
324  {
325  QDomNodeList count = m_document.elementsByTagName("item");
326 
327  if (count.count() == 0)
328  m_numResults = 0;
329  else
330  m_numResults = count.count();
331  }
332 
333  Node = itemNode.namedItem(QString("returned"));
334  if (!Node.isNull())
335  {
336  m_numReturned = Node.toElement().text().toUInt();
337  }
338  else
339  {
340  QDomNodeList items = m_document.elementsByTagName("item");
341 
342  if (items.count() == 0)
343  m_numReturned = 0;
344  else
345  m_numReturned = items.count();
346  }
347 
348  Node = itemNode.namedItem(QString("startindex"));
349  if (!Node.isNull())
350  {
351  m_numIndex = Node.toElement().text().toUInt();
352  }
353  else
354  m_numIndex = 0;
355 
356  Node = itemNode.namedItem(QString("nextpagetoken"));
357  if (!Node.isNull())
358  {
359  m_nextPageToken = Node.toElement().text();
360  }
361  else
362  m_nextPageToken = "";
363 
364  Node = itemNode.namedItem(QString("prevpagetoken"));
365  if (!Node.isNull())
366  {
367  m_prevPageToken = Node.toElement().text();
368  }
369  else
370  m_prevPageToken = "";
371 }
372 
374 {
375  if (exitcode == GENERIC_EXIT_TIMEOUT)
376  {
377  LOG(VB_GENERAL, LOG_WARNING, LOC + "Internet Search Timeout");
378 
379  if (m_searchProcess)
380  {
381  m_searchProcess->Term(true);
382  m_searchProcess->deleteLater();
383  m_searchProcess = nullptr;
384  }
385  emit searchTimedOut(this);
386  return;
387  }
388 
389  if (exitcode != GENERIC_EXIT_OK)
390  {
391  m_document.setContent(QString());
392  }
393  else
394  {
395  LOG(VB_GENERAL, LOG_INFO, LOC +
396  "Internet Search Successfully Completed");
397 
399  m_document.setContent(m_data, true);
400  }
401 
402  m_searchProcess->deleteLater();
403  m_searchProcess = nullptr;
404  emit finishedSearch(this);
405 }
406 
408 {
410 }
411 
412 void Search::SetData(QByteArray data)
413 {
414  m_data = std::move(data);
415  m_document.setContent(m_data, true);
416 
417 }
needsUpdate
bool needsUpdate(GrabberScript *script, std::chrono::hours updateFreq)
Definition: netutils.cpp:330
build_compdb.args
args
Definition: build_compdb.py:11
GrabberScript::GetType
const ArticleType & GetType() const
Definition: netgrabbermanager.h:39
Search::m_searchProcess
MythSystemLegacy * m_searchProcess
Definition: netgrabbermanager.h:176
GrabberScript::m_lock
QRecursiveMutex m_lock
Definition: netgrabbermanager.h:63
Search::finishedSearch
void finishedSearch(Search *item)
MThread::start
void start(QThread::Priority p=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:286
hardwareprofile.smolt.timeout
float timeout
Definition: smolt.py:103
GrabberScript::m_type
ArticleType m_type
Definition: netgrabbermanager.h:68
GENERIC_EXIT_OK
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
insertTreeArticleInDB
bool insertTreeArticleInDB(const QString &feedtitle, const QString &path, const QString &paththumb, ResultItem *item, ArticleType type)
Definition: netutils.cpp:399
Search::m_numIndex
uint m_numIndex
Definition: netgrabbermanager.h:184
MythSystemLegacy::Term
void Term(bool force=false)
Definition: mythsystemlegacy.cpp:285
MythSystemLegacy
Definition: mythsystemlegacy.h:67
Search::m_numResults
uint m_numResults
Definition: netgrabbermanager.h:182
Search::~Search
~Search() override
Definition: netgrabbermanager.cpp:251
GrabberScript::m_author
QString m_author
Definition: netgrabbermanager.h:69
MThread::wait
bool wait(std::chrono::milliseconds time=std::chrono::milliseconds::max())
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:303
GrabberManager::startTimer
void startTimer()
Definition: netgrabbermanager.cpp:159
GrabberManager::m_updateFreq
std::chrono::hours m_updateFreq
Definition: netgrabbermanager.h:105
GrabberScript::m_search
bool m_search
Definition: netgrabbermanager.h:70
Search::m_numReturned
uint m_numReturned
Definition: netgrabbermanager.h:183
Search::m_videoList
ResultItem::resultList m_videoList
Definition: netgrabbermanager.h:180
GrabberManager::GrabberManager
GrabberManager()
Definition: netgrabbermanager.cpp:146
MythSystemLegacy::finished
void finished(void)
GrabberScript::GrabberScript
GrabberScript(const QString &title, const QString &image, ArticleType type, const QString &author, bool search, bool tree, const QString &description, const QString &commandline, double version)
Definition: netgrabbermanager.cpp:22
GrabberScript::GetTitle
const QString & GetTitle() const
Definition: netgrabbermanager.h:37
GrabberScript::m_commandline
QString m_commandline
Definition: netgrabbermanager.h:73
GrabberScript::parseDBTree
void parseDBTree(const QString &feedtitle, const QString &path, const QString &pathThumb, QDomElement &domElem, ArticleType type)
Definition: netgrabbermanager.cpp:99
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23
MThread::RunProlog
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:196
MythSystemLegacy::ReadAll
QByteArray & ReadAll()
Definition: mythsystemlegacy.cpp:402
Search::m_nextPageToken
QString m_nextPageToken
Definition: netgrabbermanager.h:186
Search::SetData
void SetData(QByteArray data)
Definition: netgrabbermanager.cpp:412
GrabberDownloadThread::GrabberDownloadThread
GrabberDownloadThread(QObject *parent)
Definition: netgrabbermanager.cpp:190
GrabberDownloadThread::m_refreshAll
bool m_refreshAll
Definition: netgrabbermanager.h:144
mythdirs.h
findAllDBTreeGrabbers
GrabberScript::scriptList findAllDBTreeGrabbers()
Definition: netutils.cpp:115
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:14
GrabberScript::m_description
QString m_description
Definition: netgrabbermanager.h:72
markTreeUpdated
bool markTreeUpdated(GrabberScript *script, const QDateTime &curTime)
Definition: netutils.cpp:314
MythSystemLegacy::SetCommand
void SetCommand(const QString &command, uint flags)
Resets an existing MythSystemLegacy object to a new command.
Definition: mythsystemlegacy.cpp:88
mythsystemlegacy.h
Parse
Definition: rssparse.h:188
mythdate.h
GrabberDownloadThread::m_parent
QObject * m_parent
Definition: netgrabbermanager.h:141
mythlogging.h
Search::m_document
QDomDocument m_document
Definition: netgrabbermanager.h:179
Search::process
void process(void)
Definition: netgrabbermanager.cpp:301
netgrabbermanager.h
MythSystemLegacy::error
void error(uint status)
Search::m_prevPageToken
QString m_prevPageToken
Definition: netgrabbermanager.h:187
MythSystemLegacy::ShellEscape
static QString ShellEscape(const QString &in)
Definition: mythsystemlegacy.cpp:482
MythCoreContext::GetDurSetting
std::enable_if_t< std::chrono::__is_duration< T >::value, T > GetDurSetting(const QString &key, T defaultval=T::zero())
Definition: mythcorecontext.h:172
GrabberManager::m_lock
QRecursiveMutex m_lock
Definition: netgrabbermanager.h:101
netutils.h
MythSystemLegacy::Wait
uint Wait(std::chrono::seconds timeout=0s)
Definition: mythsystemlegacy.cpp:243
GENERIC_EXIT_CMD_NOT_FOUND
#define GENERIC_EXIT_CMD_NOT_FOUND
Command not found.
Definition: exitcodes.h:12
MThread::RunEpilog
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:209
gdt
GrabberDownloadThread * gdt
Definition: mythfillnetvision.cpp:27
GrabberScript
Definition: netgrabbermanager.h:19
GrabberUpdateEvent
Definition: netgrabbermanager.h:112
GrabberScript::run
void run(void) override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
Definition: netgrabbermanager.cpp:45
GrabberScript::m_title
QString m_title
Definition: netgrabbermanager.h:66
GENERIC_EXIT_TIMEOUT
#define GENERIC_EXIT_TIMEOUT
Process timed out.
Definition: exitcodes.h:24
uint
unsigned int uint
Definition: compat.h:79
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:59
GrabberScript::m_image
QString m_image
Definition: netgrabbermanager.h:67
GrabberScript::m_version
double m_version
Definition: netgrabbermanager.h:74
kMSRunShell
@ kMSRunShell
run process through shell
Definition: mythsystem.h:43
GrabberScript::m_tree
bool m_tree
Definition: netgrabbermanager.h:71
ArticleType
ArticleType
Definition: rssparse.h:20
kMSRunBackground
@ kMSRunBackground
run child in the background
Definition: mythsystem.h:38
GrabberDownloadThread::refreshAll
void refreshAll()
Definition: netgrabbermanager.cpp:210
GrabberManager::m_refreshAll
bool m_refreshAll
Definition: netgrabbermanager.h:107
GrabberManager::m_timer
QTimer * m_timer
Definition: netgrabbermanager.h:103
GrabberManager::~GrabberManager
~GrabberManager() override
Definition: netgrabbermanager.cpp:154
GrabberManager::doUpdate
void doUpdate()
Definition: netgrabbermanager.cpp:169
GrabberScript::~GrabberScript
~GrabberScript() override
Definition: netgrabbermanager.cpp:40
clearTreeItems
bool clearTreeItems(const QString &feedcommand)
Definition: netutils.cpp:360
GrabberDownloadThread::run
void run() override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
Definition: netgrabbermanager.cpp:219
Search::Search
Search()
Definition: netgrabbermanager.cpp:246
Search::m_data
QByteArray m_data
Definition: netgrabbermanager.h:178
Parse::ParseItem
static ResultItem * ParseItem(const QDomElement &item)
Definition: rssparse.cpp:742
Search::searchTimedOut
void searchTimedOut(Search *item)
GrabberDownloadThread::m_mutex
QMutex m_mutex
Definition: netgrabbermanager.h:143
GrabberDownloadThread::~GrabberDownloadThread
~GrabberDownloadThread() override
Definition: netgrabbermanager.cpp:196
Search::resetSearch
void resetSearch(void)
Definition: netgrabbermanager.cpp:295
GrabberScript::finished
void finished(void)
Search::slotProcessSearchExit
void slotProcessSearchExit(void)
Definition: netgrabbermanager.cpp:407
GrabberManager::timeout
void timeout(void)
Definition: netgrabbermanager.cpp:179
GrabberManager::stopTimer
void stopTimer()
Definition: netgrabbermanager.cpp:164
mythcontext.h
MThread
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:48
MThread::isRunning
bool isRunning(void) const
Definition: mthread.cpp:266
ResultItem::resultList
QList< ResultItem * > resultList
Definition: rssparse.h:114
GrabberDownloadThread::m_scripts
QList< GrabberScript * > m_scripts
Definition: netgrabbermanager.h:142
Parse::parseRSS
static ResultItem::resultList parseRSS(const QDomDocument &domDoc)
Definition: rssparse.cpp:719
MythSystemLegacy::Run
void Run(std::chrono::seconds timeout=0s)
Runs a command inside the /bin/sh shell. Returns immediately.
Definition: mythsystemlegacy.cpp:213
GrabberDownloadThread::cancel
void cancel()
Definition: netgrabbermanager.cpp:202
GrabberManager::refreshAll
void refreshAll()
Definition: netgrabbermanager.cpp:185
exitcodes.h
Search::executeSearch
void executeSearch(const QString &script, const QString &query, const QString &pagenum="")
Definition: netgrabbermanager.cpp:260
kMSStdOut
@ kMSStdOut
allow access to stdout
Definition: mythsystem.h:41
nv_python_libs.bbciplayer.bbciplayer_api.version
string version
Definition: bbciplayer_api.py:77
GrabberDownloadThread::finished
void finished()
GrabberDownloadThread
Definition: netgrabbermanager.h:120
LOC
#define LOC
Definition: netgrabbermanager.cpp:18