MythTV  master
netgrabbermanager.cpp
Go to the documentation of this file.
1 #include "netgrabbermanager.h"
2 
3 // qt
4 #include <QCoreApplication>
5 #include <QDir>
6 #include <QFile>
7 #include <QString>
8 #include <utility>
9 
10 #include "libmythbase/exitcodes.h"
11 #include "libmythbase/mythcorecontext.h" // for GetDurSetting TODO: excise database from MythCoreContext
12 #include "libmythbase/mythdate.h"
13 #include "libmythbase/mythdirs.h"
16 
17 #include "netutils.h"
18 
19 #define LOC QString("NetContent: ")
20 
21 // ---------------------------------------------------
22 
24 {
25  wait();
26 }
27 
29 {
30  RunProlog();
31  QMutexLocker locker(&m_lock);
32 
33  QString commandline = m_commandline;
34  MythSystemLegacy getTree(commandline, QStringList("-T"),
36  getTree.Run(15min);
37  uint status = getTree.Wait();
38 
39  if( status == GENERIC_EXIT_CMD_NOT_FOUND )
40  {
41  LOG(VB_GENERAL, LOG_ERR, LOC +
42  QString("Internet Content Source %1 cannot run, file missing.")
43  .arg(m_title));
44  }
45  else if( status == GENERIC_EXIT_OK )
46  {
47  LOG(VB_GENERAL, LOG_INFO, LOC +
48  QString("Internet Content Source %1 completed download, "
49  "beginning processing...").arg(m_title));
50 
51  QByteArray result = getTree.ReadAll();
52 
53  QDomDocument domDoc;
54  domDoc.setContent(result, true);
55  QDomElement root = domDoc.documentElement();
56  QDomElement channel = root.firstChildElement("channel");
57 
59 
60  while (!channel.isNull())
61  {
62  parseDBTree(m_title, QString(), QString(), channel, GetType());
63  channel = channel.nextSiblingElement("channel");
64  }
66  LOG(VB_GENERAL, LOG_INFO, LOC +
67  QString("Internet Content Source %1 completed processing, "
68  "marking as updated.").arg(m_title));
69  }
70  else
71  {
72  LOG(VB_GENERAL, LOG_ERR, LOC +
73  QString("Internet Content Source %1 crashed while grabbing tree.")
74  .arg(m_title));
75  }
76 
77  // NOLINTNEXTLINE(readability-misleading-indentation)
78  emit finished();
79  RunEpilog();
80 }
81 
82 void GrabberScript::parseDBTree(const QString &feedtitle, const QString &path,
83  const QString &pathThumb, QDomElement& domElem,
84  const ArticleType type)
85 {
86  QMutexLocker locker(&m_lock);
87 
88  Parse parse;
89  ResultItem::resultList articles;
90 
91  // File Handling
92  QDomElement fileitem = domElem.firstChildElement("item");
93  while (!fileitem.isNull())
94  { // Fill the article list...
95  articles.append(Parse::ParseItem(fileitem));
96  fileitem = fileitem.nextSiblingElement("item");
97  }
98 
99  while (!articles.isEmpty())
100  { // Insert the articles in the DB...
101  insertTreeArticleInDB(feedtitle, path,
102  pathThumb, articles.takeFirst(), type);
103  }
104 
105  // Directory Handling
106  QDomElement diritem = domElem.firstChildElement("directory");
107  while (!diritem.isNull())
108  {
109  QDomElement subfolder = diritem;
110  QString dirname = diritem.attribute("name");
111  QString dirthumb = diritem.attribute("thumbnail");
112  dirname.replace("/", "|");
113  QString pathToUse;
114 
115  if (path.isEmpty())
116  pathToUse = dirname;
117  else
118  pathToUse = QString("%1/%2").arg(path, dirname);
119 
120  parseDBTree(feedtitle,
121  pathToUse,
122  dirthumb,
123  subfolder,
124  type);
125  diritem = diritem.nextSiblingElement("directory");
126  }
127 }
128 
130  : m_updateFreq(gCoreContext->GetDurSetting<std::chrono::hours>("netsite.updateFreq", 24h))
131 {
132  m_timer = new QTimer();
133  connect( m_timer, &QTimer::timeout,
134  this, &GrabberManager::timeout);
135 }
136 
138 {
139  delete m_timer;
140 }
141 
143 {
144  m_timer->start(m_updateFreq);
145 }
146 
148 {
149  m_timer->stop();
150 }
151 
153 {
154  auto *gdt = new GrabberDownloadThread(this);
155  if (m_refreshAll)
156  gdt->refreshAll();
157  gdt->start(QThread::LowPriority);
158 
159  m_timer->start(m_updateFreq);
160 }
161 
163 {
164  QMutexLocker locker(&m_lock);
165  doUpdate();
166 }
167 
169 {
170  m_refreshAll = true;
171 }
172 
174  MThread("GrabberDownload")
175 {
176  m_parent = parent;
177 }
178 
180 {
181  cancel();
182  wait();
183 }
184 
186 {
187  m_mutex.lock();
188  qDeleteAll(m_scripts);
189  m_scripts.clear();
190  m_mutex.unlock();
191 }
192 
194 {
195  m_mutex.lock();
196  m_refreshAll = true;
197  if (!isRunning())
198  start();
199  m_mutex.unlock();
200 }
201 
203 {
204  RunProlog();
205 
207  auto updateFreq = gCoreContext->GetDurSetting<std::chrono::hours>(
208  "netsite.updateFreq", 24h);
209 
210  while (!m_scripts.isEmpty())
211  {
212  GrabberScript *script = m_scripts.takeFirst();
213  if (script && (needsUpdate(script, updateFreq) || m_refreshAll))
214  {
215  LOG(VB_GENERAL, LOG_INFO, LOC +
216  QString("Internet Content Source %1 Updating...")
217  .arg(script->GetTitle()));
218  script->run();
219  }
220  delete script;
221  }
222  emit finished();
223  if (m_parent)
224  QCoreApplication::postEvent(m_parent, new GrabberUpdateEvent());
225 
226  RunEpilog();
227 }
228 
230 {
231  m_videoList.clear();
232 }
233 
235 {
236  resetSearch();
237 
238  delete m_searchProcess;
239  m_searchProcess = nullptr;
240 }
241 
242 
243 void Search::executeSearch(const QString &script, const QString &query,
244  const QString &pagenum)
245 {
246  resetSearch();
247 
248  LOG(VB_GENERAL, LOG_DEBUG, "Search::executeSearch");
250 
252  this, qOverload<>(&Search::slotProcessSearchExit));
254  this, qOverload<uint>(&Search::slotProcessSearchExit));
255 
256  const QString& cmd = script;
257 
258  QStringList args;
259 
260  if (!pagenum.isEmpty())
261  {
262  args.append(QString("-p"));
263  args.append(pagenum);
264  }
265 
266  args.append("-S");
267  const QString& term = query;
268  args.append(MythSystemLegacy::ShellEscape(term));
269 
270  LOG(VB_GENERAL, LOG_INFO, LOC +
271  QString("Internet Search Query: %1 %2").arg(cmd, args.join(" ")));
272 
274  m_searchProcess->SetCommand(cmd, args, flags);
275  m_searchProcess->Run(40s);
276 }
277 
279 {
280  qDeleteAll(m_videoList);
281  m_videoList.clear();
282 }
283 
285 {
286  Parse parse;
288 
289  QDomNodeList entries = m_document.elementsByTagName("channel");
290 
291  if (entries.count() == 0)
292  {
293  m_numResults = 0;
294  m_numReturned = 0;
295  m_numIndex = 0;
296  return;
297  }
298 
299  QDomNode itemNode = entries.item(0);
300 
301  QDomNode Node = itemNode.namedItem(QString("numresults"));
302  if (!Node.isNull())
303  {
304  m_numResults = Node.toElement().text().toUInt();
305  }
306  else
307  {
308  QDomNodeList count = m_document.elementsByTagName("item");
309 
310  if (count.count() == 0)
311  m_numResults = 0;
312  else
313  m_numResults = count.count();
314  }
315 
316  Node = itemNode.namedItem(QString("returned"));
317  if (!Node.isNull())
318  {
319  m_numReturned = Node.toElement().text().toUInt();
320  }
321  else
322  {
323  QDomNodeList items = m_document.elementsByTagName("item");
324 
325  if (items.count() == 0)
326  m_numReturned = 0;
327  else
328  m_numReturned = items.count();
329  }
330 
331  Node = itemNode.namedItem(QString("startindex"));
332  if (!Node.isNull())
333  {
334  m_numIndex = Node.toElement().text().toUInt();
335  }
336  else
337  m_numIndex = 0;
338 
339  Node = itemNode.namedItem(QString("nextpagetoken"));
340  if (!Node.isNull())
341  {
342  m_nextPageToken = Node.toElement().text();
343  }
344  else
345  m_nextPageToken = "";
346 
347  Node = itemNode.namedItem(QString("prevpagetoken"));
348  if (!Node.isNull())
349  {
350  m_prevPageToken = Node.toElement().text();
351  }
352  else
353  m_prevPageToken = "";
354 }
355 
357 {
358  if (exitcode == GENERIC_EXIT_TIMEOUT)
359  {
360  LOG(VB_GENERAL, LOG_WARNING, LOC + "Internet Search Timeout");
361 
362  if (m_searchProcess)
363  {
364  m_searchProcess->Term(true);
365  m_searchProcess->deleteLater();
366  m_searchProcess = nullptr;
367  }
368  emit searchTimedOut(this);
369  return;
370  }
371 
372  if (exitcode != GENERIC_EXIT_OK)
373  {
374  m_document.setContent(QString());
375  }
376  else
377  {
378  LOG(VB_GENERAL, LOG_INFO, LOC +
379  "Internet Search Successfully Completed");
380 
382  m_document.setContent(m_data, true);
383  }
384 
385  m_searchProcess->deleteLater();
386  m_searchProcess = nullptr;
387  emit finishedSearch(this);
388 }
389 
391 {
393 }
394 
395 void Search::SetData(QByteArray data)
396 {
397  m_data = std::move(data);
398  m_document.setContent(m_data, true);
399 
400 }
needsUpdate
bool needsUpdate(GrabberScript *script, std::chrono::hours updateFreq)
Definition: netutils.cpp:330
build_compdb.args
args
Definition: build_compdb.py:11
GrabberScript::GetType
const ArticleType & GetType() const
Definition: netgrabbermanager.h:44
Search::m_searchProcess
MythSystemLegacy * m_searchProcess
Definition: netgrabbermanager.h:181
GrabberScript::m_lock
QRecursiveMutex m_lock
Definition: netgrabbermanager.h:68
Search::finishedSearch
void finishedSearch(Search *item)
MThread::start
void start(QThread::Priority p=QThread::InheritPriority)
Tell MThread to start running the thread in the near future.
Definition: mthread.cpp:283
hardwareprofile.smolt.timeout
float timeout
Definition: smolt.py:103
insertTreeArticleInDB
bool insertTreeArticleInDB(const QString &feedtitle, const QString &path, const QString &paththumb, ResultItem *item, ArticleType type)
Definition: netutils.cpp:399
Search::m_numIndex
uint m_numIndex
Definition: netgrabbermanager.h:189
MythSystemLegacy::Term
void Term(bool force=false)
Definition: mythsystemlegacy.cpp:285
GENERIC_EXIT_CMD_NOT_FOUND
@ GENERIC_EXIT_CMD_NOT_FOUND
Command not found.
Definition: exitcodes.h:13
MythSystemLegacy
Definition: mythsystemlegacy.h:67
Search::m_numResults
uint m_numResults
Definition: netgrabbermanager.h:187
Search::~Search
~Search() override
Definition: netgrabbermanager.cpp:234
GENERIC_EXIT_TIMEOUT
@ GENERIC_EXIT_TIMEOUT
Process timed out.
Definition: exitcodes.h:25
MThread::wait
bool wait(std::chrono::milliseconds time=std::chrono::milliseconds::max())
Wait for the MThread to exit, with a maximum timeout.
Definition: mthread.cpp:300
GrabberManager::startTimer
void startTimer()
Definition: netgrabbermanager.cpp:142
GrabberManager::m_updateFreq
std::chrono::hours m_updateFreq
Definition: netgrabbermanager.h:110
Search::m_numReturned
uint m_numReturned
Definition: netgrabbermanager.h:188
Search::m_videoList
ResultItem::resultList m_videoList
Definition: netgrabbermanager.h:185
GrabberManager::GrabberManager
GrabberManager()
Definition: netgrabbermanager.cpp:129
MythSystemLegacy::finished
void finished(void)
GrabberScript::GetTitle
const QString & GetTitle() const
Definition: netgrabbermanager.h:42
GrabberScript::m_commandline
QString m_commandline
Definition: netgrabbermanager.h:78
GrabberScript::parseDBTree
void parseDBTree(const QString &feedtitle, const QString &path, const QString &pathThumb, QDomElement &domElem, ArticleType type)
Definition: netgrabbermanager.cpp:82
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:39
MThread::RunProlog
void RunProlog(void)
Sets up a thread, call this if you reimplement run().
Definition: mthread.cpp:196
MythSystemLegacy::ReadAll
QByteArray & ReadAll()
Definition: mythsystemlegacy.cpp:402
Search::m_nextPageToken
QString m_nextPageToken
Definition: netgrabbermanager.h:191
Search::SetData
void SetData(QByteArray data)
Definition: netgrabbermanager.cpp:395
GrabberDownloadThread::GrabberDownloadThread
GrabberDownloadThread(QObject *parent)
Definition: netgrabbermanager.cpp:173
GrabberDownloadThread::m_refreshAll
bool m_refreshAll
Definition: netgrabbermanager.h:149
mythdirs.h
findAllDBTreeGrabbers
GrabberScript::scriptList findAllDBTreeGrabbers()
Definition: netutils.cpp:115
GENERIC_EXIT_OK
@ GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:11
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:14
markTreeUpdated
bool markTreeUpdated(GrabberScript *script, const QDateTime &curTime)
Definition: netutils.cpp:314
MythSystemLegacy::SetCommand
void SetCommand(const QString &command, uint flags)
Resets an existing MythSystemLegacy object to a new command.
Definition: mythsystemlegacy.cpp:88
mythsystemlegacy.h
Parse
Definition: rssparse.h:188
mythdate.h
GrabberDownloadThread::m_parent
QObject * m_parent
Definition: netgrabbermanager.h:146
mythlogging.h
Search::m_document
QDomDocument m_document
Definition: netgrabbermanager.h:184
Search::process
void process(void)
Definition: netgrabbermanager.cpp:284
netgrabbermanager.h
MythSystemLegacy::error
void error(uint status)
Search::m_prevPageToken
QString m_prevPageToken
Definition: netgrabbermanager.h:192
MythSystemLegacy::ShellEscape
static QString ShellEscape(const QString &in)
Definition: mythsystemlegacy.cpp:482
MythCoreContext::GetDurSetting
std::enable_if_t< std::chrono::__is_duration< T >::value, T > GetDurSetting(const QString &key, T defaultval=T::zero())
Definition: mythcorecontext.h:169
GrabberManager::m_lock
QRecursiveMutex m_lock
Definition: netgrabbermanager.h:106
netutils.h
MythSystemLegacy::Wait
uint Wait(std::chrono::seconds timeout=0s)
Definition: mythsystemlegacy.cpp:243
MThread::RunEpilog
void RunEpilog(void)
Cleans up a thread's resources, call this if you reimplement run().
Definition: mthread.cpp:209
gdt
GrabberDownloadThread * gdt
Definition: mythfillnetvision.cpp:27
GrabberScript
Definition: netgrabbermanager.h:19
GrabberUpdateEvent
Definition: netgrabbermanager.h:117
GrabberScript::run
void run(void) override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
Definition: netgrabbermanager.cpp:28
GrabberScript::m_title
QString m_title
Definition: netgrabbermanager.h:71
uint
unsigned int uint
Definition: compat.h:81
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:55
kMSRunShell
@ kMSRunShell
run process through shell
Definition: mythsystem.h:43
ArticleType
ArticleType
Definition: rssparse.h:20
kMSRunBackground
@ kMSRunBackground
run child in the background
Definition: mythsystem.h:38
GrabberDownloadThread::refreshAll
void refreshAll()
Definition: netgrabbermanager.cpp:193
GrabberManager::m_refreshAll
bool m_refreshAll
Definition: netgrabbermanager.h:112
GrabberManager::m_timer
QTimer * m_timer
Definition: netgrabbermanager.h:108
GrabberManager::~GrabberManager
~GrabberManager() override
Definition: netgrabbermanager.cpp:137
GrabberManager::doUpdate
void doUpdate()
Definition: netgrabbermanager.cpp:152
GrabberScript::~GrabberScript
~GrabberScript() override
Definition: netgrabbermanager.cpp:23
clearTreeItems
bool clearTreeItems(const QString &feedcommand)
Definition: netutils.cpp:360
GrabberDownloadThread::run
void run() override
Runs the Qt event loop unless we have a QRunnable, in which case we run the runnable run instead.
Definition: netgrabbermanager.cpp:202
mythcorecontext.h
Search::Search
Search()
Definition: netgrabbermanager.cpp:229
Search::m_data
QByteArray m_data
Definition: netgrabbermanager.h:183
Parse::ParseItem
static ResultItem * ParseItem(const QDomElement &item)
Definition: rssparse.cpp:738
Search::searchTimedOut
void searchTimedOut(Search *item)
GrabberDownloadThread::m_mutex
QMutex m_mutex
Definition: netgrabbermanager.h:148
GrabberDownloadThread::~GrabberDownloadThread
~GrabberDownloadThread() override
Definition: netgrabbermanager.cpp:179
std
Definition: mythchrono.h:23
Search::resetSearch
void resetSearch(void)
Definition: netgrabbermanager.cpp:278
GrabberScript::finished
void finished(void)
Search::slotProcessSearchExit
void slotProcessSearchExit(void)
Definition: netgrabbermanager.cpp:390
GrabberManager::timeout
void timeout(void)
Definition: netgrabbermanager.cpp:162
GrabberManager::stopTimer
void stopTimer()
Definition: netgrabbermanager.cpp:147
MThread
This is a wrapper around QThread that does several additional things.
Definition: mthread.h:48
MThread::isRunning
bool isRunning(void) const
Definition: mthread.cpp:263
ResultItem::resultList
QList< ResultItem * > resultList
Definition: rssparse.h:114
GrabberDownloadThread::m_scripts
QList< GrabberScript * > m_scripts
Definition: netgrabbermanager.h:147
Parse::parseRSS
static ResultItem::resultList parseRSS(const QDomDocument &domDoc)
Definition: rssparse.cpp:715
MythSystemLegacy::Run
void Run(std::chrono::seconds timeout=0s)
Runs a command inside the /bin/sh shell. Returns immediately.
Definition: mythsystemlegacy.cpp:213
GrabberDownloadThread::cancel
void cancel()
Definition: netgrabbermanager.cpp:185
GrabberManager::refreshAll
void refreshAll()
Definition: netgrabbermanager.cpp:168
exitcodes.h
Search::executeSearch
void executeSearch(const QString &script, const QString &query, const QString &pagenum="")
Definition: netgrabbermanager.cpp:243
kMSStdOut
@ kMSStdOut
allow access to stdout
Definition: mythsystem.h:41
GrabberDownloadThread::finished
void finished()
GrabberDownloadThread
Definition: netgrabbermanager.h:125
LOC
#define LOC
Definition: netgrabbermanager.cpp:19