MythTV  master
netgrabbermanager.cpp
Go to the documentation of this file.
1 // qt
2 #include <QCoreApplication>
3 #include <QDir>
4 #include <QFile>
5 #include <QString>
6 #include <utility>
7 
8 #include "mythdirs.h"
9 #include "mythcontext.h"
10 #include "mythsystemlegacy.h"
11 #include "exitcodes.h"
12 #include "mythdate.h"
13 #include "mythlogging.h"
14 #include "mythsystemlegacy.h"
15 
16 #include "netgrabbermanager.h"
17 #include "netutils.h"
18 
19 #define LOC QString("NetContent: ")
20 
21 // ---------------------------------------------------
22 
23 GrabberScript::GrabberScript(const QString& title, const QString& image,
24  const ArticleType type, const QString& author,
25  const bool search, const bool tree,
26  const QString& description, const QString& commandline,
27  const double version) :
28  MThread("GrabberScript")
29 {
30  m_title = title;
31  m_image = image;
32  m_type = type;
33  m_author = author;
34  m_search = search;
35  m_tree = tree;
36  m_description = description;
37  m_commandline = commandline;
39 }
40 
42 {
43  wait();
44 }
45 
47 {
48  RunProlog();
49  QMutexLocker locker(&m_lock);
50 
51  QString commandline = m_commandline;
52  MythSystemLegacy getTree(commandline, QStringList("-T"),
54  getTree.Run(15min);
55  uint status = getTree.Wait();
56 
57  if( status == GENERIC_EXIT_CMD_NOT_FOUND )
58  {
59  LOG(VB_GENERAL, LOG_ERR, LOC +
60  QString("Internet Content Source %1 cannot run, file missing.")
61  .arg(m_title));
62  }
63  else if( status == GENERIC_EXIT_OK )
64  {
65  LOG(VB_GENERAL, LOG_INFO, LOC +
66  QString("Internet Content Source %1 completed download, "
67  "beginning processing...").arg(m_title));
68 
69  QByteArray result = getTree.ReadAll();
70 
71  QDomDocument domDoc;
72  domDoc.setContent(result, true);
73  QDomElement root = domDoc.documentElement();
74  QDomElement channel = root.firstChildElement("channel");
75 
77 
78  while (!channel.isNull())
79  {
80  parseDBTree(m_title, QString(), QString(), channel, GetType());
81  channel = channel.nextSiblingElement("channel");
82  }
84  LOG(VB_GENERAL, LOG_INFO, LOC +
85  QString("Internet Content Source %1 completed processing, "
86  "marking as updated.").arg(m_title));
87  }
88  else
89  {
90  LOG(VB_GENERAL, LOG_ERR, LOC +
91  QString("Internet Content Source %1 crashed while grabbing tree.")
92  .arg(m_title));
93  }
94 
95  // NOLINTNEXTLINE(readability-misleading-indentation)
96  emit finished();
97  RunEpilog();
98 }
99 
100 void GrabberScript::parseDBTree(const QString &feedtitle, const QString &path,
101  const QString &pathThumb, QDomElement& domElem,
102  const ArticleType type)
103 {
104  QMutexLocker locker(&m_lock);
105 
106  Parse parse;
107  ResultItem::resultList articles;
108 
109  // File Handling
110  QDomElement fileitem = domElem.firstChildElement("item");
111  while (!fileitem.isNull())
112  { // Fill the article list...
113  articles.append(Parse::ParseItem(fileitem));
114  fileitem = fileitem.nextSiblingElement("item");
115  }
116 
117  while (!articles.isEmpty())
118  { // Insert the articles in the DB...
119  insertTreeArticleInDB(feedtitle, path,
120  pathThumb, articles.takeFirst(), type);
121  }
122 
123  // Directory Handling
124  QDomElement diritem = domElem.firstChildElement("directory");
125  while (!diritem.isNull())
126  {
127  QDomElement subfolder = diritem;
128  QString dirname = diritem.attribute("name");
129  QString dirthumb = diritem.attribute("thumbnail");
130  dirname.replace("/", "|");
131  QString pathToUse;
132 
133  if (path.isEmpty())
134  pathToUse = dirname;
135  else
136  pathToUse = QString("%1/%2").arg(path, dirname);
137 
138  parseDBTree(feedtitle,
139  pathToUse,
140  dirthumb,
141  subfolder,
142  type);
143  diritem = diritem.nextSiblingElement("directory");
144  }
145 }
146 
148 {
149  m_updateFreq = gCoreContext->GetDurSetting<std::chrono::hours>("netsite.updateFreq", 24h);
150  m_timer = new QTimer();
151  connect( m_timer, &QTimer::timeout,
152  this, &GrabberManager::timeout);
153 }
154 
156 {
157  delete m_timer;
158 }
159 
161 {
162  m_timer->start(m_updateFreq);
163 }
164 
166 {
167  m_timer->stop();
168 }
169 
171 {
172  auto *gdt = new GrabberDownloadThread(this);
173  if (m_refreshAll)
174  gdt->refreshAll();
175  gdt->start(QThread::LowPriority);
176 
177  m_timer->start(m_updateFreq);
178 }
179 
181 {
182  QMutexLocker locker(&m_lock);
183  doUpdate();
184 }
185 
187 {
188  m_refreshAll = true;
189 }
190 
192  MThread("GrabberDownload")
193 {
194  m_parent = parent;
195 }
196 
198 {
199  cancel();
200  wait();
201 }
202 
204 {
205  m_mutex.lock();
206  qDeleteAll(m_scripts);
207  m_scripts.clear();
208  m_mutex.unlock();
209 }
210 
212 {
213  m_mutex.lock();
214  m_refreshAll = true;
215  if (!isRunning())
216  start();
217  m_mutex.unlock();
218 }
219 
221 {
222  RunProlog();
223 
225  auto updateFreq = gCoreContext->GetDurSetting<std::chrono::hours>(
226  "netsite.updateFreq", 24h);
227 
228  while (!m_scripts.isEmpty())
229  {
230  GrabberScript *script = m_scripts.takeFirst();
231  if (script && (needsUpdate(script, updateFreq) || m_refreshAll))
232  {
233  LOG(VB_GENERAL, LOG_INFO, LOC +
234  QString("Internet Content Source %1 Updating...")
235  .arg(script->GetTitle()));
236  script->run();
237  }
238  delete script;
239  }
240  emit finished();
241  if (m_parent)
242  QCoreApplication::postEvent(m_parent, new GrabberUpdateEvent());
243 
244  RunEpilog();
245 }
246 
248 {
249  m_videoList.clear();
250 }
251 
253 {
254  resetSearch();
255 
256  delete m_searchProcess;
257  m_searchProcess = nullptr;
258 }
259 
260 
261 void Search::executeSearch(const QString &script, const QString &query,
262  const QString &pagenum)
263 {
264  resetSearch();
265 
266  LOG(VB_GENERAL, LOG_DEBUG, "Search::executeSearch");
268 
270  this, qOverload<>(&Search::slotProcessSearchExit));
272  this, qOverload<uint>(&Search::slotProcessSearchExit));
273 
274  const QString& cmd = script;
275 
276  QStringList args;
277 
278  if (!pagenum.isEmpty())
279  {
280  args.append(QString("-p"));
281  args.append(pagenum);
282  }
283 
284  args.append("-S");
285  const QString& term = query;
286  args.append(MythSystemLegacy::ShellEscape(term));
287 
288  LOG(VB_GENERAL, LOG_INFO, LOC +
289  QString("Internet Search Query: %1 %2").arg(cmd, args.join(" ")));
290 
292  m_searchProcess->SetCommand(cmd, args, flags);
293  m_searchProcess->Run(40s);
294 }
295 
297 {
298  qDeleteAll(m_videoList);
299  m_videoList.clear();
300 }
301 
303 {
304  Parse parse;
306 
307  QDomNodeList entries = m_document.elementsByTagName("channel");
308 
309  if (entries.count() == 0)
310  {
311  m_numResults = 0;
312  m_numReturned = 0;
313  m_numIndex = 0;
314  return;
315  }
316 
317  QDomNode itemNode = entries.item(0);
318 
319  QDomNode Node = itemNode.namedItem(QString("numresults"));
320  if (!Node.isNull())
321  {
322  m_numResults = Node.toElement().text().toUInt();
323  }
324  else
325  {
326  QDomNodeList count = m_document.elementsByTagName("item");
327 
328  if (count.count() == 0)
329  m_numResults = 0;
330  else
331  m_numResults = count.count();
332  }
333 
334  Node = itemNode.namedItem(QString("returned"));
335  if (!Node.isNull())
336  {
337  m_numReturned = Node.toElement().text().toUInt();
338  }
339  else
340  {
341  QDomNodeList items = m_document.elementsByTagName("item");
342 
343  if (items.count() == 0)
344  m_numReturned = 0;
345  else
346  m_numReturned = items.count();
347  }
348 
349  Node = itemNode.namedItem(QString("startindex"));
350  if (!Node.isNull())
351  {
352  m_numIndex = Node.toElement().text().toUInt();
353  }
354  else
355  m_numIndex = 0;
356 
357  Node = itemNode.namedItem(QString("nextpagetoken"));
358  if (!Node.isNull())
359  {
360  m_nextPageToken = Node.toElement().text();
361  }
362  else
363  m_nextPageToken = "";
364 
365  Node = itemNode.namedItem(QString("prevpagetoken"));
366  if (!Node.isNull())
367  {
368  m_prevPageToken = Node.toElement().text();
369  }
370  else
371  m_prevPageToken = "";
372 }
373 
375 {
376  if (exitcode == GENERIC_EXIT_TIMEOUT)
377  {
378  LOG(VB_GENERAL, LOG_WARNING, LOC + "Internet Search Timeout");
379 
380  if (m_searchProcess)
381  {
382  m_searchProcess->Term(true);
383  m_searchProcess->deleteLater();
384  m_searchProcess = nullptr;
385  }
386  emit searchTimedOut(this);
387  return;
388  }
389 
390  if (exitcode != GENERIC_EXIT_OK)
391  {
392  m_document.setContent(QString());
393  }
394  else
395  {
396  LOG(VB_GENERAL, LOG_INFO, LOC +
397  "Internet Search Successfully Completed");
398 
400  m_document.setContent(m_data, true);
401  }
402 
403  m_searchProcess->deleteLater();
404  m_searchProcess = nullptr;
405  emit finishedSearch(this);
406 }
407 
409 {
411 }
412 
413 void Search::SetData(QByteArray data)
414 {
415  m_data = std::move(data);
416  m_document.setContent(m_data, true);
417 
418 }
needsUpdate
bool needsUpdate(GrabberScript *script, std::chrono::hours updateFreq)
Definition: netutils.cpp:330
build_compdb.args
args
Definition: build_compdb.py:11
GrabberScript::GetType
const ArticleType & GetType() const
Definition: netgrabbermanager.h:39
Search::m_searchProcess
MythSystemLegacy * m_searchProcess
Definition: netgrabbermanager.h:176
GrabberScript::m_lock
QRecursiveMutex m_lock
Definition: netgrabbermanager.h:63
Search::finishedSearch
void finishedSearch(Search *item)
MThread::start
void start(QThread::Priority p=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:286
hardwareprofile.smolt.timeout
float timeout
Definition: smolt.py:103
GrabberScript::m_type
ArticleType m_type
Definition: netgrabbermanager.h:68
GENERIC_EXIT_OK
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
insertTreeArticleInDB
bool insertTreeArticleInDB(const QString &feedtitle, const QString &path, const QString &paththumb, ResultItem *item, ArticleType type)
Definition: netutils.cpp:399
Search::m_numIndex
uint m_numIndex
Definition: netgrabbermanager.h:184
MythSystemLegacy::Term
void Term(bool force=false)
Definition: mythsystemlegacy.cpp:284
MythSystemLegacy
Definition: mythsystemlegacy.h:67
Search::m_numResults
uint m_numResults
Definition: netgrabbermanager.h:182
Search::~Search
~Search() override
Definition: netgrabbermanager.cpp:252
GrabberScript::m_author
QString m_author
Definition: netgrabbermanager.h:69
MThread::wait
bool wait(std::chrono::milliseconds time=std::chrono::milliseconds::max())
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:303
GrabberManager::startTimer
void startTimer()
Definition: netgrabbermanager.cpp:160
GrabberManager::m_updateFreq
std::chrono::hours m_updateFreq
Definition: netgrabbermanager.h:105
GrabberScript::m_search
bool m_search
Definition: netgrabbermanager.h:70
Search::m_numReturned
uint m_numReturned
Definition: netgrabbermanager.h:183
Search::m_videoList
ResultItem::resultList m_videoList
Definition: netgrabbermanager.h:180
GrabberManager::GrabberManager
GrabberManager()
Definition: netgrabbermanager.cpp:147
MythSystemLegacy::finished
void finished(void)
GrabberScript::GrabberScript
GrabberScript(const QString &title, const QString &image, ArticleType type, const QString &author, bool search, bool tree, const QString &description, const QString &commandline, double version)
Definition: netgrabbermanager.cpp:23
GrabberScript::GetTitle
const QString & GetTitle() const
Definition: netgrabbermanager.h:37
GrabberScript::m_commandline
QString m_commandline
Definition: netgrabbermanager.h:73
GrabberScript::parseDBTree
void parseDBTree(const QString &feedtitle, const QString &path, const QString &pathThumb, QDomElement &domElem, ArticleType type)
Definition: netgrabbermanager.cpp:100
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23
MThread::RunProlog
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:196
MythSystemLegacy::ReadAll
QByteArray & ReadAll()
Definition: mythsystemlegacy.cpp:401
Search::m_nextPageToken
QString m_nextPageToken
Definition: netgrabbermanager.h:186
Search::SetData
void SetData(QByteArray data)
Definition: netgrabbermanager.cpp:413
GrabberDownloadThread::GrabberDownloadThread
GrabberDownloadThread(QObject *parent)
Definition: netgrabbermanager.cpp:191
GrabberDownloadThread::m_refreshAll
bool m_refreshAll
Definition: netgrabbermanager.h:144
mythdirs.h
findAllDBTreeGrabbers
GrabberScript::scriptList findAllDBTreeGrabbers()
Definition: netutils.cpp:115
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
GrabberScript::m_description
QString m_description
Definition: netgrabbermanager.h:72
markTreeUpdated
bool markTreeUpdated(GrabberScript *script, const QDateTime &curTime)
Definition: netutils.cpp:314
MythSystemLegacy::SetCommand
void SetCommand(const QString &command, uint flags)
Resets an existing MythSystemLegacy object to a new command.
Definition: mythsystemlegacy.cpp:87
mythsystemlegacy.h
Parse
Definition: rssparse.h:188
mythdate.h
GrabberDownloadThread::m_parent
QObject * m_parent
Definition: netgrabbermanager.h:141
mythlogging.h
Search::m_document
QDomDocument m_document
Definition: netgrabbermanager.h:179
Search::process
void process(void)
Definition: netgrabbermanager.cpp:302
netgrabbermanager.h
MythSystemLegacy::error
void error(uint status)
Search::m_prevPageToken
QString m_prevPageToken
Definition: netgrabbermanager.h:187
MythSystemLegacy::ShellEscape
static QString ShellEscape(const QString &in)
Definition: mythsystemlegacy.cpp:481
MythCoreContext::GetDurSetting
std::enable_if_t< std::chrono::__is_duration< T >::value, T > GetDurSetting(const QString &key, T defaultval=T::zero())
Definition: mythcorecontext.h:170
GrabberManager::m_lock
QRecursiveMutex m_lock
Definition: netgrabbermanager.h:101
netutils.h
MythSystemLegacy::Wait
uint Wait(std::chrono::seconds timeout=0s)
Definition: mythsystemlegacy.cpp:242
GENERIC_EXIT_CMD_NOT_FOUND
#define GENERIC_EXIT_CMD_NOT_FOUND
Command not found.
Definition: exitcodes.h:12
MThread::RunEpilog
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:209
GrabberScript
Definition: netgrabbermanager.h:19
GrabberUpdateEvent
Definition: netgrabbermanager.h:112
GrabberScript::run
void run(void) override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
Definition: netgrabbermanager.cpp:46
GrabberScript::m_title
QString m_title
Definition: netgrabbermanager.h:66
GENERIC_EXIT_TIMEOUT
#define GENERIC_EXIT_TIMEOUT
Process timed out.
Definition: exitcodes.h:24
uint
unsigned int uint
Definition: compat.h:144
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:60
GrabberScript::m_image
QString m_image
Definition: netgrabbermanager.h:67
GrabberScript::m_version
double m_version
Definition: netgrabbermanager.h:74
kMSRunShell
@ kMSRunShell
run process through shell
Definition: mythsystem.h:43
GrabberScript::m_tree
bool m_tree
Definition: netgrabbermanager.h:71
ArticleType
ArticleType
Definition: rssparse.h:20
kMSRunBackground
@ kMSRunBackground
run child in the background
Definition: mythsystem.h:38
GrabberDownloadThread::refreshAll
void refreshAll()
Definition: netgrabbermanager.cpp:211
GrabberManager::m_refreshAll
bool m_refreshAll
Definition: netgrabbermanager.h:107
GrabberManager::m_timer
QTimer * m_timer
Definition: netgrabbermanager.h:103
GrabberManager::~GrabberManager
~GrabberManager() override
Definition: netgrabbermanager.cpp:155
gdt
GrabberDownloadThread * gdt
Definition: mythplugins/mythnetvision/mythfillnetvision/main.cpp:26
GrabberManager::doUpdate
void doUpdate()
Definition: netgrabbermanager.cpp:170
GrabberScript::~GrabberScript
~GrabberScript() override
Definition: netgrabbermanager.cpp:41
clearTreeItems
bool clearTreeItems(const QString &feedcommand)
Definition: netutils.cpp:360
GrabberDownloadThread::run
void run() override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
Definition: netgrabbermanager.cpp:220
Search::Search
Search()
Definition: netgrabbermanager.cpp:247
Search::m_data
QByteArray m_data
Definition: netgrabbermanager.h:178
Parse::ParseItem
static ResultItem * ParseItem(const QDomElement &item)
Definition: rssparse.cpp:741
Search::searchTimedOut
void searchTimedOut(Search *item)
GrabberDownloadThread::m_mutex
QMutex m_mutex
Definition: netgrabbermanager.h:143
GrabberDownloadThread::~GrabberDownloadThread
~GrabberDownloadThread() override
Definition: netgrabbermanager.cpp:197
Search::resetSearch
void resetSearch(void)
Definition: netgrabbermanager.cpp:296
GrabberScript::finished
void finished(void)
Search::slotProcessSearchExit
void slotProcessSearchExit(void)
Definition: netgrabbermanager.cpp:408
GrabberManager::timeout
void timeout(void)
Definition: netgrabbermanager.cpp:180
GrabberManager::stopTimer
void stopTimer()
Definition: netgrabbermanager.cpp:165
mythcontext.h
MThread
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:48
MThread::isRunning
bool isRunning(void) const
Definition: mthread.cpp:266
ResultItem::resultList
QList< ResultItem * > resultList
Definition: rssparse.h:114
GrabberDownloadThread::m_scripts
QList< GrabberScript * > m_scripts
Definition: netgrabbermanager.h:142
Parse::parseRSS
static ResultItem::resultList parseRSS(const QDomDocument &domDoc)
Definition: rssparse.cpp:718
MythSystemLegacy::Run
void Run(std::chrono::seconds timeout=0s)
Runs a command inside the /bin/sh shell. Returns immediately.
Definition: mythsystemlegacy.cpp:212
GrabberDownloadThread::cancel
void cancel()
Definition: netgrabbermanager.cpp:203
GrabberManager::refreshAll
void refreshAll()
Definition: netgrabbermanager.cpp:186
exitcodes.h
Search::executeSearch
void executeSearch(const QString &script, const QString &query, const QString &pagenum="")
Definition: netgrabbermanager.cpp:261
kMSStdOut
@ kMSStdOut
allow access to stdout
Definition: mythsystem.h:41
nv_python_libs.bbciplayer.bbciplayer_api.version
string version
Definition: bbciplayer_api.py:81
GrabberDownloadThread::finished
void finished()
GrabberDownloadThread
Definition: netgrabbermanager.h:120
LOC
#define LOC
Definition: netgrabbermanager.cpp:19