MythTV  master
netgrabbermanager.cpp
Go to the documentation of this file.
1 // qt
2 #include <QCoreApplication>
3 #include <QDir>
4 #include <QFile>
5 #include <QString>
6 #include <utility>
7 
8 #include "mythdirs.h"
9 #include "mythcontext.h"
10 #include "mythsystemlegacy.h"
11 #include "exitcodes.h"
12 #include "mythdate.h"
13 #include "mythlogging.h"
14 #include "mythsystemlegacy.h"
15 
16 #include "netgrabbermanager.h"
17 #include "netutils.h"
18 
19 #define LOC QString("NetContent: ")
20 
21 using namespace std;
22 
23 // ---------------------------------------------------
24 
25 GrabberScript::GrabberScript(const QString& title, const QString& image,
26  const ArticleType &type, const QString& author,
27  const bool& search, const bool& tree,
28  const QString& description, const QString& commandline,
29  const double& version) :
30  MThread("GrabberScript")
31 {
32  m_title = title;
33  m_image = image;
34  m_type = type;
35  m_author = author;
36  m_search = search;
37  m_tree = tree;
38  m_description = description;
39  m_commandline = commandline;
41 }
42 
44 {
45  wait();
46 }
47 
49 {
50  RunProlog();
51  QMutexLocker locker(&m_lock);
52 
53  QString commandline = m_commandline;
54  MythSystemLegacy getTree(commandline, QStringList("-T"),
56  getTree.Run(900);
57  uint status = getTree.Wait();
58 
59  if( status == GENERIC_EXIT_CMD_NOT_FOUND )
60  {
61  LOG(VB_GENERAL, LOG_ERR, LOC +
62  QString("Internet Content Source %1 cannot run, file missing.")
63  .arg(m_title));
64  }
65  else if( status == GENERIC_EXIT_OK )
66  {
67  LOG(VB_GENERAL, LOG_INFO, LOC +
68  QString("Internet Content Source %1 completed download, "
69  "beginning processing...").arg(m_title));
70 
71  QByteArray result = getTree.ReadAll();
72 
73  QDomDocument domDoc;
74  domDoc.setContent(result, true);
75  QDomElement root = domDoc.documentElement();
76  QDomElement channel = root.firstChildElement("channel");
77 
79 
80  while (!channel.isNull())
81  {
82  parseDBTree(m_title, QString(), QString(), channel, GetType());
83  channel = channel.nextSiblingElement("channel");
84  }
86  LOG(VB_GENERAL, LOG_INFO, LOC +
87  QString("Internet Content Source %1 completed processing, "
88  "marking as updated.").arg(m_title));
89  }
90  else
91  {
92  LOG(VB_GENERAL, LOG_ERR, LOC +
93  QString("Internet Content Source %1 crashed while grabbing tree.")
94  .arg(m_title));
95  }
96 
97  // NOLINTNEXTLINE(readability-misleading-indentation)
98  emit finished();
99  RunEpilog();
100 }
101 
102 void GrabberScript::parseDBTree(const QString &feedtitle, const QString &path,
103  const QString &pathThumb, QDomElement& domElem,
104  const ArticleType &type)
105 {
106  QMutexLocker locker(&m_lock);
107 
108  Parse parse;
109  ResultItem::resultList articles;
110 
111  // File Handling
112  QDomElement fileitem = domElem.firstChildElement("item");
113  while (!fileitem.isNull())
114  { // Fill the article list...
115  articles.append(parse.ParseItem(fileitem));
116  fileitem = fileitem.nextSiblingElement("item");
117  }
118 
119  while (!articles.isEmpty())
120  { // Insert the articles in the DB...
121  insertTreeArticleInDB(feedtitle, path,
122  pathThumb, articles.takeFirst(), type);
123  }
124 
125  // Directory Handling
126  QDomElement diritem = domElem.firstChildElement("directory");
127  while (!diritem.isNull())
128  {
129  QDomElement subfolder = diritem;
130  QString dirname = diritem.attribute("name");
131  QString dirthumb = diritem.attribute("thumbnail");
132  dirname.replace("/", "|");
133  QString pathToUse;
134 
135  if (path.isEmpty())
136  pathToUse = dirname;
137  else
138  pathToUse = QString("%1/%2").arg(path).arg(dirname);
139 
140  parseDBTree(feedtitle,
141  pathToUse,
142  dirthumb,
143  subfolder,
144  type);
145  diritem = diritem.nextSiblingElement("directory");
146  }
147 }
148 
150 {
152  "netsite.updateFreq", 24) * 3600 * 1000);
153  m_timer = new QTimer();
154  connect( m_timer, SIGNAL(timeout()),
155  this, SLOT(timeout()));
156 }
157 
159 {
160  delete m_timer;
161 }
162 
164 {
165  m_timer->start(m_updateFreq);
166 }
167 
169 {
170  m_timer->stop();
171 }
172 
174 {
175  auto *gdt = new GrabberDownloadThread(this);
176  if (m_refreshAll)
177  gdt->refreshAll();
178  gdt->start(QThread::LowPriority);
179 
180  m_timer->start(m_updateFreq);
181 }
182 
184 {
185  QMutexLocker locker(&m_lock);
186  doUpdate();
187 }
188 
190 {
191  m_refreshAll = true;
192 }
193 
195  MThread("GrabberDownload")
196 {
197  m_parent = parent;
198 }
199 
201 {
202  cancel();
203  wait();
204 }
205 
207 {
208  m_mutex.lock();
209  qDeleteAll(m_scripts);
210  m_scripts.clear();
211  m_mutex.unlock();
212 }
213 
215 {
216  m_mutex.lock();
217  m_refreshAll = true;
218  if (!isRunning())
219  start();
220  m_mutex.unlock();
221 }
222 
224 {
225  RunProlog();
226 
228  uint updateFreq = gCoreContext->GetNumSetting(
229  "netsite.updateFreq", 24);
230 
231  while (m_scripts.count())
232  {
233  GrabberScript *script = m_scripts.takeFirst();
234  if (script && (needsUpdate(script, updateFreq) || m_refreshAll))
235  {
236  LOG(VB_GENERAL, LOG_INFO, LOC +
237  QString("Internet Content Source %1 Updating...")
238  .arg(script->GetTitle()));
239  script->run();
240  }
241  delete script;
242  }
243  emit finished();
244  if (m_parent)
245  QCoreApplication::postEvent(m_parent, new GrabberUpdateEvent());
246 
247  RunEpilog();
248 }
249 
251 {
252  m_videoList.clear();
253 }
254 
256 {
257  resetSearch();
258 
259  delete m_searchProcess;
260  m_searchProcess = nullptr;
261 }
262 
263 
264 void Search::executeSearch(const QString &script, const QString &query,
265  const QString &pagenum)
266 {
267  resetSearch();
268 
269  LOG(VB_GENERAL, LOG_DEBUG, "Search::executeSearch");
271 
272  connect(m_searchProcess, SIGNAL(finished()),
273  this, SLOT(slotProcessSearchExit()));
274  connect(m_searchProcess, SIGNAL(error(uint)),
275  this, SLOT(slotProcessSearchExit(uint)));
276 
277  const QString& cmd = script;
278 
279  QStringList args;
280 
281  if (!pagenum.isEmpty())
282  {
283  args.append(QString("-p"));
284  args.append(pagenum);
285  }
286 
287  args.append("-S");
288  const QString& term = query;
289  args.append(MythSystemLegacy::ShellEscape(term));
290 
291  LOG(VB_GENERAL, LOG_INFO, LOC +
292  QString("Internet Search Query: %1 %2").arg(cmd).arg(args.join(" ")));
293 
295  m_searchProcess->SetCommand(cmd, args, flags);
296  m_searchProcess->Run(40);
297 }
298 
300 {
301  qDeleteAll(m_videoList);
302  m_videoList.clear();
303 }
304 
306 {
307  Parse parse;
309 
310  QDomNodeList entries = m_document.elementsByTagName("channel");
311 
312  if (entries.count() == 0)
313  {
314  m_numResults = 0;
315  m_numReturned = 0;
316  m_numIndex = 0;
317  return;
318  }
319 
320  QDomNode itemNode = entries.item(0);
321 
322  QDomNode Node = itemNode.namedItem(QString("numresults"));
323  if (!Node.isNull())
324  {
325  m_numResults = Node.toElement().text().toUInt();
326  }
327  else
328  {
329  QDomNodeList count = m_document.elementsByTagName("item");
330 
331  if (count.count() == 0)
332  m_numResults = 0;
333  else
334  m_numResults = count.count();
335  }
336 
337  Node = itemNode.namedItem(QString("returned"));
338  if (!Node.isNull())
339  {
340  m_numReturned = Node.toElement().text().toUInt();
341  }
342  else
343  {
344  QDomNodeList items = m_document.elementsByTagName("item");
345 
346  if (items.count() == 0)
347  m_numReturned = 0;
348  else
349  m_numReturned = items.count();
350  }
351 
352  Node = itemNode.namedItem(QString("startindex"));
353  if (!Node.isNull())
354  {
355  m_numIndex = Node.toElement().text().toUInt();
356  }
357  else
358  m_numIndex = 0;
359 
360  Node = itemNode.namedItem(QString("nextpagetoken"));
361  if (!Node.isNull())
362  {
363  m_nextPageToken = Node.toElement().text();
364  }
365  else
366  m_nextPageToken = "";
367 
368  Node = itemNode.namedItem(QString("prevpagetoken"));
369  if (!Node.isNull())
370  {
371  m_prevPageToken = Node.toElement().text();
372  }
373  else
374  m_prevPageToken = "";
375 }
376 
378 {
379  if (exitcode == GENERIC_EXIT_TIMEOUT)
380  {
381  LOG(VB_GENERAL, LOG_WARNING, LOC + "Internet Search Timeout");
382 
383  if (m_searchProcess)
384  {
385  m_searchProcess->Term(true);
386  m_searchProcess->deleteLater();
387  m_searchProcess = nullptr;
388  }
389  emit searchTimedOut(this);
390  return;
391  }
392 
393  if (exitcode != GENERIC_EXIT_OK)
394  {
395  m_document.setContent(QString());
396  }
397  else
398  {
399  LOG(VB_GENERAL, LOG_INFO, LOC +
400  "Internet Search Successfully Completed");
401 
403  m_document.setContent(m_data, true);
404  }
405 
406  m_searchProcess->deleteLater();
407  m_searchProcess = nullptr;
408  emit finishedSearch(this);
409 }
410 
411 void Search::SetData(QByteArray data)
412 {
413  m_data = std::move(data);
414  m_document.setContent(m_data, true);
415 
416 }
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:215
void Run(time_t timeout=0)
Runs a command inside the /bin/sh shell. Returns immediately.
uint m_numReturned
QString m_nextPageToken
#define LOC
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:46
const ArticleType & GetType() const
bool insertTreeArticleInDB(const QString &feedtitle, const QString &path, const QString &paththumb, ResultItem *item, ArticleType type)
Definition: netutils.cpp:399
void run() override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
void SetData(QByteArray data)
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
#define GENERIC_EXIT_TIMEOUT
Process timed out.
Definition: exitcodes.h:24
static void error(const char *str,...)
Definition: vbi.c:42
void finished(void)
bool wait(unsigned long time=ULONG_MAX)
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:311
const QString & GetTitle() const
~GrabberManager() override
static QString ShellEscape(const QString &in)
MythSystemLegacy * m_searchProcess
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
ResultItem * ParseItem(const QDomElement &item) const
Definition: rssparse.cpp:735
GrabberScript::scriptList findAllDBTreeGrabbers()
Definition: netutils.cpp:115
QString m_prevPageToken
uint m_numResults
QList< ResultItem * > resultList
Definition: rssparse.h:114
ResultItem::resultList m_videoList
ArticleType
Definition: rssparse.h:20
void Term(bool force=false)
GrabberScript(const QString &title, const QString &image, const ArticleType &type, const QString &author, const bool &search, const bool &tree, const QString &description, const QString &commandline, const double &version)
#define GENERIC_EXIT_CMD_NOT_FOUND
Command not found.
Definition: exitcodes.h:12
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
QByteArray & ReadAll()
bool isRunning(void) const
Definition: mthread.cpp:274
QDomDocument m_document
unsigned int uint
Definition: compat.h:140
void process(void)
GrabberDownloadThread * gdt
void run(void) override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
QList< GrabberScript * > m_scripts
uint Wait(time_t timeout=0)
run process through shell
Definition: mythsystem.h:41
bool needsUpdate(GrabberScript *script, uint updateFreq)
Definition: netutils.cpp:330
int GetNumSetting(const QString &key, int defaultval=0)
GrabberDownloadThread(QObject *parent)
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
ResultItem::resultList parseRSS(const QDomDocument &domDoc)
Definition: rssparse.cpp:712
void executeSearch(const QString &script, const QString &query, const QString &pagenum="")
bool clearTreeItems(const QString &feedcommand)
Definition: netutils.cpp:360
void slotProcessSearchExit(uint exitcode=0)
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:202
void start(QThread::Priority p=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:294
run child in the background
Definition: mythsystem.h:36
void parseDBTree(const QString &feedtitle, const QString &path, const QString &pathThumb, QDomElement &domElem, const ArticleType &type)
void searchTimedOut(Search *item)
void finishedSearch(Search *item)
void resetSearch(void)
~GrabberScript() override
bool markTreeUpdated(GrabberScript *script, const QDateTime &curTime)
Definition: netutils.cpp:314
QByteArray m_data
~Search() override
allow access to stdout
Definition: mythsystem.h:39
void SetCommand(const QString &command, uint flags)
Resets an existing MythSystemLegacy object to a new command.
ArticleType m_type