MythTV  master
netgrabbermanager.cpp
Go to the documentation of this file.
1 // qt
2 #include <QCoreApplication>
3 #include <QDir>
4 #include <QFile>
5 #include <QString>
6 #include <utility>
7 
8 #include "mythdirs.h"
9 #include "mythcontext.h"
10 #include "mythsystemlegacy.h"
11 #include "exitcodes.h"
12 #include "mythdate.h"
13 #include "mythlogging.h"
14 #include "mythsystemlegacy.h"
15 
16 #include "netgrabbermanager.h"
17 #include "netutils.h"
18 
19 #define LOC QString("NetContent: ")
20 
21 using namespace std;
22 
23 // ---------------------------------------------------
24 
25 GrabberScript::GrabberScript(const QString& title, const QString& image,
26  const ArticleType &type, const QString& author,
27  const bool& search, const bool& tree,
28  const QString& description, const QString& commandline,
29  const double& version) :
30  MThread("GrabberScript")
31 {
32  m_title = title;
33  m_image = image;
34  m_type = type;
35  m_author = author;
36  m_search = search;
37  m_tree = tree;
38  m_description = description;
39  m_commandline = commandline;
41 }
42 
44 {
45  wait();
46 }
47 
49 {
50  RunProlog();
51  QMutexLocker locker(&m_lock);
52 
53  QString commandline = m_commandline;
54  MythSystemLegacy getTree(commandline, QStringList("-T"),
56  getTree.Run(900);
57  uint status = getTree.Wait();
58 
59  if( status == GENERIC_EXIT_CMD_NOT_FOUND )
60  LOG(VB_GENERAL, LOG_ERR, LOC +
61  QString("Internet Content Source %1 cannot run, file missing.")
62  .arg(m_title));
63  else if( status == GENERIC_EXIT_OK )
64  {
65  LOG(VB_GENERAL, LOG_INFO, LOC +
66  QString("Internet Content Source %1 completed download, "
67  "beginning processing...").arg(m_title));
68 
69  QByteArray result = getTree.ReadAll();
70 
71  QDomDocument domDoc;
72  domDoc.setContent(result, true);
73  QDomElement root = domDoc.documentElement();
74  QDomElement channel = root.firstChildElement("channel");
75 
77 
78  while (!channel.isNull())
79  {
80  parseDBTree(m_title, QString(), QString(), channel, GetType());
81  channel = channel.nextSiblingElement("channel");
82  }
84  LOG(VB_GENERAL, LOG_INFO, LOC +
85  QString("Internet Content Source %1 completed processing, "
86  "marking as updated.").arg(m_title));
87  }
88  else
89  LOG(VB_GENERAL, LOG_ERR, LOC +
90  QString("Internet Content Source %1 crashed while grabbing tree.")
91  .arg(m_title));
92 
93  emit finished();
94  RunEpilog();
95 }
96 
97 void GrabberScript::parseDBTree(const QString &feedtitle, const QString &path,
98  const QString &pathThumb, QDomElement& domElem,
99  const ArticleType &type)
100 {
101  QMutexLocker locker(&m_lock);
102 
103  Parse parse;
104  ResultItem::resultList articles;
105 
106  // File Handling
107  QDomElement fileitem = domElem.firstChildElement("item");
108  while (!fileitem.isNull())
109  { // Fill the article list...
110  articles.append(parse.ParseItem(fileitem));
111  fileitem = fileitem.nextSiblingElement("item");
112  }
113 
114  while (!articles.isEmpty())
115  { // Insert the articles in the DB...
116  insertTreeArticleInDB(feedtitle, path,
117  pathThumb, articles.takeFirst(), type);
118  }
119 
120  // Directory Handling
121  QDomElement diritem = domElem.firstChildElement("directory");
122  while (!diritem.isNull())
123  {
124  QDomElement subfolder = diritem;
125  QString dirname = diritem.attribute("name");
126  QString dirthumb = diritem.attribute("thumbnail");
127  dirname.replace("/", "|");
128  QString pathToUse;
129 
130  if (path.isEmpty())
131  pathToUse = dirname;
132  else
133  pathToUse = QString("%1/%2").arg(path).arg(dirname);
134 
135  parseDBTree(feedtitle,
136  pathToUse,
137  dirthumb,
138  subfolder,
139  type);
140  diritem = diritem.nextSiblingElement("directory");
141  }
142 }
143 
145 {
147  "netsite.updateFreq", 24) * 3600 * 1000);
148  m_timer = new QTimer();
149  connect( m_timer, SIGNAL(timeout()),
150  this, SLOT(timeout()));
151 }
152 
154 {
155  delete m_timer;
156 }
157 
159 {
160  m_timer->start(m_updateFreq);
161 }
162 
164 {
165  m_timer->stop();
166 }
167 
169 {
171  if (m_refreshAll)
172  gdt->refreshAll();
173  gdt->start(QThread::LowPriority);
174 
175  m_timer->start(m_updateFreq);
176 }
177 
179 {
180  QMutexLocker locker(&m_lock);
181  doUpdate();
182 }
183 
185 {
186  m_refreshAll = true;
187 }
188 
190  MThread("GrabberDownload")
191 {
192  m_parent = parent;
193 }
194 
196 {
197  cancel();
198  wait();
199 }
200 
202 {
203  m_mutex.lock();
204  qDeleteAll(m_scripts);
205  m_scripts.clear();
206  m_mutex.unlock();
207 }
208 
210 {
211  m_mutex.lock();
212  m_refreshAll = true;
213  if (!isRunning())
214  start();
215  m_mutex.unlock();
216 }
217 
219 {
220  RunProlog();
221 
223  uint updateFreq = gCoreContext->GetNumSetting(
224  "netsite.updateFreq", 24);
225 
226  while (m_scripts.count())
227  {
228  GrabberScript *script = m_scripts.takeFirst();
229  if (script && (needsUpdate(script, updateFreq) || m_refreshAll))
230  {
231  LOG(VB_GENERAL, LOG_INFO, LOC +
232  QString("Internet Content Source %1 Updating...")
233  .arg(script->GetTitle()));
234  script->run();
235  }
236  delete script;
237  }
238  emit finished();
239  if (m_parent)
240  QCoreApplication::postEvent(m_parent, new GrabberUpdateEvent());
241 
242  RunEpilog();
243 }
244 
246 {
247  m_videoList.clear();
248 }
249 
251 {
252  resetSearch();
253 
254  delete m_searchProcess;
255  m_searchProcess = nullptr;
256 }
257 
258 
259 void Search::executeSearch(const QString &script, const QString &query,
260  const QString &pagenum)
261 {
262  resetSearch();
263 
264  LOG(VB_GENERAL, LOG_DEBUG, "Search::executeSearch");
266 
267  connect(m_searchProcess, SIGNAL(finished()),
268  this, SLOT(slotProcessSearchExit()));
269  connect(m_searchProcess, SIGNAL(error(uint)),
270  this, SLOT(slotProcessSearchExit(uint)));
271 
272  const QString& cmd = script;
273 
274  QStringList args;
275 
276  if (!pagenum.isEmpty())
277  {
278  args.append(QString("-p"));
279  args.append(pagenum);
280  }
281 
282  args.append("-S");
283  const QString& term = query;
284  args.append(MythSystemLegacy::ShellEscape(term));
285 
286  LOG(VB_GENERAL, LOG_INFO, LOC +
287  QString("Internet Search Query: %1 %2").arg(cmd).arg(args.join(" ")));
288 
290  m_searchProcess->SetCommand(cmd, args, flags);
291  m_searchProcess->Run(40);
292 }
293 
295 {
296  qDeleteAll(m_videoList);
297  m_videoList.clear();
298 }
299 
301 {
302  Parse parse;
304 
305  QDomNodeList entries = m_document.elementsByTagName("channel");
306 
307  if (entries.count() == 0)
308  {
309  m_numResults = 0;
310  m_numReturned = 0;
311  m_numIndex = 0;
312  return;
313  }
314 
315  QDomNode itemNode = entries.item(0);
316 
317  QDomNode Node = itemNode.namedItem(QString("numresults"));
318  if (!Node.isNull())
319  {
320  m_numResults = Node.toElement().text().toUInt();
321  }
322  else
323  {
324  QDomNodeList count = m_document.elementsByTagName("item");
325 
326  if (count.count() == 0)
327  m_numResults = 0;
328  else
329  m_numResults = count.count();
330  }
331 
332  Node = itemNode.namedItem(QString("returned"));
333  if (!Node.isNull())
334  {
335  m_numReturned = Node.toElement().text().toUInt();
336  }
337  else
338  {
339  QDomNodeList items = m_document.elementsByTagName("item");
340 
341  if (items.count() == 0)
342  m_numReturned = 0;
343  else
344  m_numReturned = items.count();
345  }
346 
347  Node = itemNode.namedItem(QString("startindex"));
348  if (!Node.isNull())
349  {
350  m_numIndex = Node.toElement().text().toUInt();
351  }
352  else
353  m_numIndex = 0;
354 
355  Node = itemNode.namedItem(QString("nextpagetoken"));
356  if (!Node.isNull())
357  {
358  m_nextPageToken = Node.toElement().text();
359  }
360  else
361  m_nextPageToken = "";
362 
363  Node = itemNode.namedItem(QString("prevpagetoken"));
364  if (!Node.isNull())
365  {
366  m_prevPageToken = Node.toElement().text();
367  }
368  else
369  m_prevPageToken = "";
370 }
371 
373 {
374  if (exitcode == GENERIC_EXIT_TIMEOUT)
375  {
376  LOG(VB_GENERAL, LOG_WARNING, LOC + "Internet Search Timeout");
377 
378  if (m_searchProcess)
379  {
380  m_searchProcess->Term(true);
381  m_searchProcess->deleteLater();
382  m_searchProcess = nullptr;
383  }
384  emit searchTimedOut(this);
385  return;
386  }
387 
388  if (exitcode != GENERIC_EXIT_OK)
389  {
390  m_document.setContent(QString());
391  }
392  else
393  {
394  LOG(VB_GENERAL, LOG_INFO, LOC +
395  "Internet Search Successfully Completed");
396 
398  m_document.setContent(m_data, true);
399  }
400 
401  m_searchProcess->deleteLater();
402  m_searchProcess = nullptr;
403  emit finishedSearch(this);
404 }
405 
406 void Search::SetData(QByteArray data)
407 {
408  m_data = std::move(data);
409  m_document.setContent(m_data, true);
410 
411 }
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:215
void start(QThread::Priority=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:294
void Run(time_t timeout=0)
Runs a command inside the /bin/sh shell. Returns immediately.
uint m_numReturned
QString m_nextPageToken
#define LOC
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:46
const ArticleType & GetType() const
allow access to stdout
Definition: mythsystem.h:39
bool insertTreeArticleInDB(const QString &feedtitle, const QString &path, const QString &paththumb, ResultItem *item, ArticleType type)
Definition: netutils.cpp:395
void run() override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
void SetData(QByteArray data)
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
#define GENERIC_EXIT_TIMEOUT
Process timed out.
Definition: exitcodes.h:24
static void error(const char *str,...)
Definition: vbi.c:42
void finished(void)
bool wait(unsigned long time=ULONG_MAX)
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:311
const QString & GetTitle() const
run child in the background
Definition: mythsystem.h:36
static QString ShellEscape(const QString &in)
MythSystemLegacy * m_searchProcess
unsigned int uint
Definition: compat.h:140
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
ResultItem * ParseItem(const QDomElement &item) const
Definition: rssparse.cpp:735
GrabberScript::scriptList findAllDBTreeGrabbers()
Definition: netutils.cpp:115
QString m_prevPageToken
uint m_numResults
void SetCommand(const QString &, uint)
Resets an existing MythSystemLegacy object to a new command.
ResultItem::resultList m_videoList
void Term(bool force=false)
GrabberScript(const QString &title, const QString &image, const ArticleType &type, const QString &author, const bool &search, const bool &tree, const QString &description, const QString &commandline, const double &version)
run process through shell
Definition: mythsystem.h:41
#define GENERIC_EXIT_CMD_NOT_FOUND
Command not found.
Definition: exitcodes.h:12
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
QByteArray & ReadAll()
bool isRunning(void) const
Definition: mthread.cpp:274
QDomDocument m_document
void process(void)
GrabberDownloadThread * gdt
void run(void) override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
QList< GrabberScript * > m_scripts
uint Wait(time_t timeout=0)
bool needsUpdate(GrabberScript *script, uint updateFreq)
Definition: netutils.cpp:326
int GetNumSetting(const QString &key, int defaultval=0)
GrabberDownloadThread(QObject *parent)
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
ResultItem::resultList parseRSS(const QDomDocument &domDoc)
Definition: rssparse.cpp:712
void executeSearch(const QString &script, const QString &query, const QString &pagenum="")
bool clearTreeItems(const QString &feedcommand)
Definition: netutils.cpp:356
void slotProcessSearchExit(uint exitcode=0)
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:202
void parseDBTree(const QString &feedtitle, const QString &path, const QString &pathThumb, QDomElement &domElem, const ArticleType &type)
void searchTimedOut(Search *item)
void finishedSearch(Search *item)
QList< ResultItem * > resultList
Definition: rssparse.h:114
void resetSearch(void)
bool markTreeUpdated(GrabberScript *script, const QDateTime &curTime)
Definition: netutils.cpp:310
QByteArray m_data
enum ArticleTypes ArticleType
ArticleType m_type