MythTV  master
metadatagrabber.cpp
Go to the documentation of this file.
1 // Qt headers
2 #include <QDateTime>
3 #include <QDir>
4 #include <QMap>
5 #include <QMutex>
6 #include <QMutexLocker>
7 #include <QRegExp>
8 #include <utility>
9 
10 // MythTV headers
11 #include "metadatagrabber.h"
12 #include "metadatacommon.h"
13 #include "mythsystemlegacy.h"
14 #include "exitcodes.h"
15 #include "mythdate.h"
16 #include "mythdirs.h"
17 #include "mythlogging.h"
18 #include "mythcorecontext.h"
19 
20 #define LOC QString("Metadata Grabber: ")
21 #define kGrabberRefresh 60
22 
24 static QMutex grabberLock;
25 static QDateTime grabberAge;
26 
27 struct GrabberOpts {
28  QString m_path;
29  QString m_setting;
30  QString m_def;
31 };
32 
33 // TODO
34 // it would be nice to statically compile these, but I can't manage to get it
35 // to compile. apparently initializer lists are supported in QT5/CPP11 that
36 // will make this work. for now, use a lock and initialize on first access.
37 // https://bugreports.qt-project.org/browse/QTBUG-25679
38 static QMap<GrabberType, GrabberOpts> grabberTypes;
39 static QMap<QString, GrabberType> grabberTypeStrings;
40 static bool initialized = false;
41 static QMutex typeLock;
42 
43 static GrabberOpts GrabberOptsMaker(QString thepath, QString thesetting, QString thedefault)
44 {
46 
47  opts.m_path = std::move(thepath);
48  opts.m_setting = std::move(thesetting);
49  opts.m_def = std::move(thedefault);
50 
51  return opts;
52 }
53 
54 static void InitializeStaticMaps(void)
55 {
56  QMutexLocker lock(&typeLock);
57 
58  if (!initialized)
59  {
61  GrabberOptsMaker ("%1metadata/Movie/",
62  "MovieGrabber",
63  "metadata/Movie/tmdb3.py" );
65  GrabberOptsMaker ( "%1metadata/Television/",
66  "TelevisionGrabber",
67  "metadata/Television/ttvdb.py" );
69  GrabberOptsMaker ( "%1metadata/Game/",
70  "mythgame.MetadataGrabber",
71  "metadata/Game/giantbomb.py" );
73  GrabberOptsMaker ( "%1metadata/Music",
74  "",
75  "" );
76 
78  grabberTypeStrings["television"] = kGrabberTelevision;
81 
82 
83  initialized = true;
84  }
85 }
86 
88 {
89  return MetaGrabberScript::GetList(kGrabberAll, refresh);
90 }
91 
92 GrabberList MetaGrabberScript::GetList(const QString &type, bool refresh)
93 {
94  QString tmptype = type.toLower();
95  if (!grabberTypeStrings.contains(tmptype))
96  // unknown type, return empty list
97  return GrabberList();
98 
99  return MetaGrabberScript::GetList(grabberTypeStrings[tmptype], refresh);
100 }
101 
103  bool refresh)
104 {
106 
107  GrabberList tmpGrabberList;
108  GrabberList retGrabberList;
109  {
110  QMutexLocker listLock(&grabberLock);
111  QDateTime now = MythDate::current();
112 
113  // refresh grabber scripts every 60 seconds
114  // this might have to be revised, or made more intelligent if
115  // the delay during refreshes is too great
116  if (refresh || !grabberAge.isValid() ||
117  (grabberAge.secsTo(now) > kGrabberRefresh))
118  {
119  grabberList.clear();
120  LOG(VB_GENERAL, LOG_DEBUG, LOC + "Clearing grabber cache");
121 
122  // loop through different types of grabber scripts and the
123  // directories they are stored in
124  QMap<GrabberType, GrabberOpts>::const_iterator it;
125  for (it = grabberTypes.begin(); it != grabberTypes.end(); ++it)
126  {
127  QString path = (it->m_path).arg(GetShareDir());
128  QStringList scripts = QDir(path).entryList(QDir::Executable | QDir::Files);
129  if (scripts.count() == 0)
130  // no scripts found
131  continue;
132 
133  // loop through discovered scripts
134  for (const auto& name : qAsConst(scripts))
135  {
136  QString cmd = QDir(path).filePath(name);
137  MetaGrabberScript script(cmd);
138 
139  if (script.IsValid())
140  {
141  LOG(VB_GENERAL, LOG_DEBUG, LOC + "Adding " + script.m_command);
142  grabberList.append(script);
143  }
144  }
145  }
146 
147  grabberAge = now;
148  }
149 
150  tmpGrabberList = grabberList;
151  }
152 
153  for (const auto& item : qAsConst(tmpGrabberList))
154  {
155  if ((type == kGrabberAll) || (item.GetType() == type))
156  retGrabberList.append(item);
157  }
158 
159  return retGrabberList;
160 }
161 
163  const MetadataLookup *lookup)
164 {
165  if (!lookup)
166  {
167  return GetType(defaultType);
168  }
169 
170  if (!lookup->GetInetref().isEmpty() &&
171  lookup->GetInetref() != "00000000")
172  {
173  // inetref is defined, see if we have a pre-defined grabber
174  MetaGrabberScript grabber = FromInetref(lookup->GetInetref());
175 
176  if (grabber.IsValid())
177  {
178  return grabber;
179  }
180  // matching grabber was not found, just use the default
181  // fall through
182  }
183 
184  return GetType(defaultType);
185 }
186 
188 {
189  QString tmptype = type.toLower();
190  if (!grabberTypeStrings.contains(tmptype))
191  // unknown type, return empty grabber
192  return MetaGrabberScript();
193 
195 }
196 
198 {
200 
201  QString cmd = gCoreContext->GetSetting(grabberTypes[type].m_setting,
202  grabberTypes[type].m_def);
203 
204  if (cmd.isEmpty())
205  {
206  // should the python bindings had not been installed at any stage
207  // the settings could have been set to an empty string, so use default
208  cmd = grabberTypes[type].m_def;
209  }
210 
211  if (grabberAge.isValid() && grabberAge.secsTo(MythDate::current()) <= kGrabberRefresh)
212  {
213  // just pull it from the cache
214  GrabberList list = GetList();
215  for (const auto& item : qAsConst(list))
216  if (item.GetPath().endsWith(cmd))
217  return item;
218  }
219 
220  // polling the cache will cause a refresh, so lets just grab and
221  // process the script directly
222  QString fullcmd = QString("%1%2").arg(GetShareDir()).arg(cmd);
223  MetaGrabberScript script(fullcmd);
224 
225  if (script.IsValid())
226  {
227  return script;
228  }
229 
230  return MetaGrabberScript();
231 }
232 
234  bool absolute)
235 {
236  GrabberList list = GetList();
237 
238  // search for direct match on tag
239  for (const auto& item : qAsConst(list))
240  {
241  if (item.GetCommand() == tag)
242  {
243  return item;
244  }
245  }
246 
247  // no direct match. do we require a direct match? search for one that works
248  if (!absolute)
249  {
250  for (const auto& item : qAsConst(list))
251  {
252  if (item.Accepts(tag))
253  {
254  return item;
255  }
256  }
257  }
258 
259  // no working match. return a blank
260  return MetaGrabberScript();
261 }
262 
264  bool absolute)
265 {
266  static QRegExp s_retagref(R"(^([a-zA-Z0-9_\-\.]+\.[a-zA-Z0-9]{1,3})_(.*))");
267  static QRegExp s_retagref2(R"(^([a-zA-Z0-9_\-\.]+\.[a-zA-Z0-9]{1,3}):(.*))");
268  static QMutex s_reLock;
269  QMutexLocker lock(&s_reLock);
270  QString tag;
271 
272  if (s_retagref.indexIn(inetref) > -1)
273  {
274  tag = s_retagref.cap(1);
275  }
276  else if (s_retagref2.indexIn(inetref) > -1)
277  {
278  tag = s_retagref2.cap(1);
279  }
280  if (!tag.isEmpty())
281  {
282  // match found, pull out the grabber
283  MetaGrabberScript script = MetaGrabberScript::FromTag(tag, absolute);
284  if (script.IsValid())
285  return script;
286  }
287 
288  // no working match, return a blank
289  return MetaGrabberScript();
290 }
291 
292 QString MetaGrabberScript::CleanedInetref(const QString &inetref)
293 {
294  static QRegExp s_retagref(R"(^([a-zA-Z0-9_\-\.]+\.[a-zA-Z0-9]{1,3})_(.*))");
295  static QRegExp s_retagref2(R"(^([a-zA-Z0-9_\-\.]+\.[a-zA-Z0-9]{1,3}):(.*))");
296  static QMutex s_reLock;
297  QMutexLocker lock(&s_reLock);
298 
299  // try to strip grabber tag from inetref
300  if (s_retagref.indexIn(inetref) > -1)
301  return s_retagref.cap(2);
302  if (s_retagref2.indexIn(inetref) > -1)
303  return s_retagref2.cap(2);
304 
305  return inetref;
306 }
307 
308 MetaGrabberScript::MetaGrabberScript(QString path, const QDomElement &dom) :
309  m_fullcommand(std::move(path))
310 {
311  ParseGrabberVersion(dom);
312 }
313 
315 {
316  ParseGrabberVersion(dom);
317 }
318 
320 {
321  if (path.isEmpty())
322  return;
323  m_fullcommand = path;
324  if (path[0] != '/')
325  m_fullcommand.prepend(QString("%1metadata").arg(GetShareDir()));
326 
327  MythSystemLegacy grabber(path, QStringList() << "-v",
329  grabber.Run();
330  if (grabber.Wait() != GENERIC_EXIT_OK)
331  // script failed
332  return;
333 
334  QByteArray result = grabber.ReadAll();
335  if (result.isEmpty())
336  // no output
337  return;
338 
339  QDomDocument doc;
340  doc.setContent(result, true);
341  QDomElement root = doc.documentElement();
342  if (root.isNull())
343  // no valid XML
344  return;
345 
347  if (m_name.isEmpty())
348  // XML not processed correctly
349  return;
350 
351  m_valid = true;
352 }
353 
355 {
356  if (this != &other)
357  {
358  m_name = other.m_name;
359  m_author = other.m_author;
360  m_thumbnail = other.m_thumbnail;
361  m_command = other.m_command;
363  m_type = other.m_type;
364  m_typestring = other.m_typestring;
366  m_accepts = other.m_accepts;
367  m_version = other.m_version;
368  m_valid = other.m_valid;
369  }
370 
371  return *this;
372 }
373 
374 
375 void MetaGrabberScript::ParseGrabberVersion(const QDomElement &item)
376 {
377  m_name = item.firstChildElement("name").text();
378  m_author = item.firstChildElement("author").text();
379  m_thumbnail = item.firstChildElement("thumbnail").text();
380  m_command = item.firstChildElement("command").text();
381  m_description = item.firstChildElement("description").text();
382  m_version = item.firstChildElement("version").text().toFloat();
383  m_typestring = item.firstChildElement("type").text().toLower();
384 
385  if (!m_typestring.isEmpty() && grabberTypeStrings.contains(m_typestring))
387  else
389 
390  QDomElement accepts = item.firstChildElement("accepts");
391  if (!accepts.isNull())
392  {
393  while (!accepts.isNull())
394  {
395  m_accepts.append(accepts.text());
396  accepts = accepts.nextSiblingElement("accepts");
397  }
398  }
399 }
400 
402 {
403  if (!m_valid || m_fullcommand.isEmpty())
404  return false;
405 
406  QStringList args; args << "-t";
408 
409  grabber.Run();
410  return grabber.Wait() == GENERIC_EXIT_OK;
411 }
412 
413 // TODO
414 // using the MetadataLookup object as both argument input, and parsed output,
415 // is clumsy. break the inputs out into a separate object, and spawn a new
416 // MetadataLookup object in ParseMetadataItem, rather than requiring an
417 // existing one to reuse.
419  MetadataLookup *lookup, bool passseas)
420 {
422  MetadataLookupList list;
423 
424  LOG(VB_GENERAL, LOG_INFO, QString("Running Grabber: %1 %2")
425  .arg(m_fullcommand).arg(args.join(" ")));
426 
427  grabber.Run();
428  if (grabber.Wait(60) != GENERIC_EXIT_OK)
429  return list;
430 
431  QByteArray result = grabber.ReadAll();
432  if (!result.isEmpty())
433  {
434  QDomDocument doc;
435  doc.setContent(result, true);
436  QDomElement root = doc.documentElement();
437  QDomElement item = root.firstChildElement("item");
438 
439  while (!item.isNull())
440  {
441  MetadataLookup *tmp = ParseMetadataItem(item, lookup, passseas);
442  tmp->SetInetref(QString("%1_%2").arg(m_command)
443  .arg(tmp->GetInetref()));
444  if (!tmp->GetCollectionref().isEmpty())
445  {
446  tmp->SetCollectionref(QString("%1_%2").arg(m_command)
447  .arg(tmp->GetCollectionref()));
448  }
449  list.append(tmp);
450  // MetadataLookup is to be owned by the list
451  tmp->DecrRef();
452  item = item.nextSiblingElement("item");
453  }
454  }
455  return list;
456 }
457 
458 QString MetaGrabberScript::GetRelPath(void) const
459 {
460  QString share = GetShareDir();
461  if (m_fullcommand.startsWith(share))
462  return m_fullcommand.right(m_fullcommand.size() - share.size());
463 
464  return QString();
465 }
466 
467 void MetaGrabberScript::toMap(InfoMap &metadataMap) const
468 {
469  metadataMap["name"] = m_name;
470  metadataMap["author"] = m_author;
471  metadataMap["thumbnailfilename"] = m_thumbnail;
472  metadataMap["command"] = m_command;
473  metadataMap["description"] = m_description;
474  metadataMap["version"] = QString::number(m_version);
475  metadataMap["type"] = m_typestring;
476 }
477 
479 {
480  args << "-l"
482  << "-a"
484 }
485 
487  MetadataLookup *lookup, bool passseas)
488 {
489  QStringList args;
491 
492  args << "-M"
493  << title;
494 
495  return RunGrabber(args, lookup, passseas);
496 }
497 
499  const QString &subtitle, MetadataLookup *lookup,
500  bool passseas)
501 {
502  QStringList args;
504 
505  args << "-N"
506  << title
507  << subtitle;
508 
509  return RunGrabber(args, lookup, passseas);
510 }
511 
513  const QString &title, const QString &subtitle,
514  MetadataLookup *lookup, bool passseas)
515 {
516  (void)title;
517  QStringList args;
519 
520  args << "-N"
521  << CleanedInetref(inetref)
522  << subtitle;
523 
524  return RunGrabber(args, lookup, passseas);
525 }
526 
528  MetadataLookup *lookup, bool passseas)
529 {
530  QStringList args;
532 
533  args << "-D"
534  << CleanedInetref(inetref);
535 
536  return RunGrabber(args, lookup, passseas);
537 }
538 
540  int season, int episode, MetadataLookup *lookup,
541  bool passseas)
542 {
543  QStringList args;
545 
546  args << "-D"
547  << CleanedInetref(inetref)
548  << QString::number(season)
549  << QString::number(episode);
550 
551  return RunGrabber(args, lookup, passseas);
552 }
553 
555  const QString &collectionref, MetadataLookup *lookup,
556  bool passseas)
557 {
558  QStringList args;
560 
561  args << "-C"
562  << CleanedInetref(collectionref);
563 
564  return RunGrabber(args, lookup, passseas);
565 }
QStringList m_accepts
void Run(time_t timeout=0)
Runs a command inside the /bin/sh shell. Returns immediately.
MetadataLookup * ParseMetadataItem(const QDomElement &item, MetadataLookup *lookup, bool passseas)
QString GetRelPath(void) const
MythLocale * GetLocale(void) const
#define LOC
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
static QMutex listLock
GrabberType m_type
static QString CleanedInetref(const QString &inetref)
QHash< QString, QString > InfoMap
Definition: mythtypes.h:15
static QDateTime grabberAge
QDomDocument doc("MYTHARCHIVEITEM")
GrabberType
MetadataLookupList Search(const QString &title, MetadataLookup *lookup, bool passseas=true)
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
static MetaGrabberScript FromTag(const QString &tag, bool absolute=false)
static guint32 * tmp
Definition: goom_core.cpp:30
MetadataLookupList LookupCollection(const QString &collectionref, MetadataLookup *lookup, bool passseas=true)
static GrabberOpts GrabberOptsMaker(QString thepath, QString thesetting, QString thedefault)
QString GetCountryCode() const
Definition: mythlocale.cpp:58
static GrabberList GetList(bool refresh=false)
static MetaGrabberScript GetGrabber(GrabberType defaultType, const MetadataLookup *lookup=nullptr)
MetadataLookupList RunGrabber(const QStringList &args, MetadataLookup *lookup, bool passseas)
MetadataLookupList SearchSubtitle(const QString &title, const QString &subtitle, MetadataLookup *lookup, bool passseas=true)
bool IsValid(void) const
void ParseGrabberVersion(const QDomElement &item)
void toMap(InfoMap &metadataMap) const
#define kGrabberRefresh
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
QString GetSetting(const QString &key, const QString &defaultval="")
QByteArray & ReadAll()
QString GetShareDir(void)
Definition: mythdirs.cpp:222
static void SetDefaultArgs(QStringList &args)
static QMap< QString, GrabberType > grabberTypeStrings
QString GetInetref() const
MetadataLookupList LookupData(const QString &inetref, MetadataLookup *lookup, bool passseas=true)
MetaGrabberScript()=default
static void InitializeStaticMaps(void)
uint Wait(time_t timeout=0)
static QMap< GrabberType, GrabberOpts > grabberTypes
static QMutex grabberLock
run process through shell
Definition: mythsystem.h:41
MetaGrabberScript & operator=(const MetaGrabberScript &other)
static bool initialized
static GrabberList grabberList
QList< MetaGrabberScript > GrabberList
QString GetLanguage(void)
Returns two character ISO-639 language descriptor for UI language.
static MetaGrabberScript FromInetref(const QString &inetref, bool absolute=false)
GrabberType GetType(void) const
allow access to stdout
Definition: mythsystem.h:39
static QMutex typeLock
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23