MythTV  master
metadatagrabber.cpp
Go to the documentation of this file.
1 // Qt headers
2 #include <QDateTime>
3 #include <QDir>
4 #include <QMap>
5 #include <QMutex>
6 #include <QMutexLocker>
7 #include <QRegExp>
8 #include <utility>
9 
10 // MythTV headers
11 #include "metadatagrabber.h"
12 #include "metadatacommon.h"
13 #include "mythsystemlegacy.h"
14 #include "exitcodes.h"
15 #include "mythdate.h"
16 #include "mythdirs.h"
17 #include "mythlogging.h"
18 #include "mythcorecontext.h"
19 
20 #define LOC QString("Metadata Grabber: ")
21 #define kGrabberRefresh 60
22 
24 static QMutex grabberLock;
25 static QDateTime grabberAge;
26 
27 typedef struct GrabberOpts {
28  QString path;
29  QString setting;
30  QString def;
31 } GrabberOpts;
32 
33 // TODO
34 // it would be nice to statically compile these, but I can't manage to get it
35 // to compile. apparently initializer lists are supported in QT5/CPP11 that
36 // will make this work. for now, use a lock and initialize on first access.
37 // https://bugreports.qt-project.org/browse/QTBUG-25679
38 static QMap<GrabberType, GrabberOpts> grabberTypes;
39 static QMap<QString, GrabberType> grabberTypeStrings;
40 static bool initialized = false;
41 static QMutex typeLock;
42 
43 static GrabberOpts GrabberOptsMaker(QString thepath, QString thesetting, QString thedefault)
44 {
46 
47  opts.path = std::move(thepath);
48  opts.setting = std::move(thesetting);
49  opts.def = std::move(thedefault);
50 
51  return opts;
52 }
53 
54 static void InitializeStaticMaps(void)
55 {
56  QMutexLocker lock(&typeLock);
57 
58  if (!initialized)
59  {
61  GrabberOptsMaker ("%1metadata/Movie/",
62  "MovieGrabber",
63  "metadata/Movie/tmdb3.py" );
65  GrabberOptsMaker ( "%1metadata/Television/",
66  "TelevisionGrabber",
67  "metadata/Television/ttvdb.py" );
69  GrabberOptsMaker ( "%1metadata/Game/",
70  "mythgame.MetadataGrabber",
71  "metadata/Game/giantbomb.py" );
73  GrabberOptsMaker ( "%1metadata/Music",
74  "",
75  "" );
76 
78  grabberTypeStrings["television"] = kGrabberTelevision;
81 
82 
83  initialized = true;
84  }
85 }
86 
88 {
89  return MetaGrabberScript::GetList(kGrabberAll, refresh);
90 }
91 
92 GrabberList MetaGrabberScript::GetList(const QString &type, bool refresh)
93 {
94  QString tmptype = type.toLower();
95  if (!grabberTypeStrings.contains(tmptype))
96  // unknown type, return empty list
97  return GrabberList();
98 
99  return MetaGrabberScript::GetList(grabberTypeStrings[tmptype], refresh);
100 }
101 
103  bool refresh)
104 {
106 
107  GrabberList tmpGrabberList, retGrabberList;
108  {
109  QMutexLocker listLock(&grabberLock);
110  QDateTime now = MythDate::current();
111 
112  // refresh grabber scripts every 60 seconds
113  // this might have to be revised, or made more intelligent if
114  // the delay during refreshes is too great
115  if (refresh || !grabberAge.isValid() ||
116  (grabberAge.secsTo(now) > kGrabberRefresh))
117  {
118  grabberList.clear();
119  LOG(VB_GENERAL, LOG_DEBUG, LOC + "Clearing grabber cache");
120 
121  // loop through different types of grabber scripts and the
122  // directories they are stored in
123  QMap<GrabberType, GrabberOpts>::const_iterator it;
124  for (it = grabberTypes.begin(); it != grabberTypes.end(); ++it)
125  {
126  QString path = (it->path).arg(GetShareDir());
127  QStringList scripts = QDir(path).entryList(QDir::Executable | QDir::Files);
128  if (scripts.count() == 0)
129  // no scripts found
130  continue;
131 
132  // loop through discovered scripts
133  QStringList::const_iterator it2 = scripts.begin();
134  for (; it2 != scripts.end(); ++it2)
135  {
136  QString cmd = QDir(path).filePath(*it2);
137  MetaGrabberScript script(cmd);
138 
139  if (script.IsValid())
140  {
141  LOG(VB_GENERAL, LOG_DEBUG, LOC + "Adding " + script.m_command);
142  grabberList.append(script);
143  }
144  }
145  }
146 
147  grabberAge = now;
148  }
149 
150  tmpGrabberList = grabberList;
151  }
152 
153  GrabberList::const_iterator it = tmpGrabberList.begin();
154  for (; it != tmpGrabberList.end(); ++it)
155  {
156  if ((type == kGrabberAll) || (it->GetType() == type))
157  retGrabberList.append(*it);
158  }
159 
160  return retGrabberList;
161 }
162 
164  const MetadataLookup *lookup)
165 {
166  if (!lookup)
167  {
168  return GetType(defaultType);
169  }
170 
171  if (!lookup->GetInetref().isEmpty() &&
172  lookup->GetInetref() != "00000000")
173  {
174  // inetref is defined, see if we have a pre-defined grabber
175  MetaGrabberScript grabber = FromInetref(lookup->GetInetref());
176 
177  if (grabber.IsValid())
178  {
179  return grabber;
180  }
181  // matching grabber was not found, just use the default
182  // fall through
183  }
184 
185  return GetType(defaultType);
186 }
187 
189 {
190  QString tmptype = type.toLower();
191  if (!grabberTypeStrings.contains(tmptype))
192  // unknown type, return empty grabber
193  return MetaGrabberScript();
194 
196 }
197 
199 {
201 
202  QString cmd = gCoreContext->GetSetting(grabberTypes[type].setting,
203  grabberTypes[type].def);
204 
205  if (cmd.isEmpty())
206  {
207  // should the python bindings had not been installed at any stage
208  // the settings could have been set to an empty string, so use default
209  cmd = grabberTypes[type].def;
210  }
211 
212  if (grabberAge.isValid() && grabberAge.secsTo(MythDate::current()) <= kGrabberRefresh)
213  {
214  // just pull it from the cache
215  GrabberList list = GetList();
216  GrabberList::const_iterator it = list.begin();
217 
218  for (; it != list.end(); ++it)
219  if (it->GetPath().endsWith(cmd))
220  return *it;
221  }
222 
223  // polling the cache will cause a refresh, so lets just grab and
224  // process the script directly
225  QString fullcmd = QString("%1%2").arg(GetShareDir()).arg(cmd);
226  MetaGrabberScript script(fullcmd);
227 
228  if (script.IsValid())
229  {
230  return script;
231  }
232 
233  return MetaGrabberScript();
234 }
235 
237  bool absolute)
238 {
239  GrabberList list = GetList();
240  GrabberList::const_iterator it = list.begin();
241 
242  // search for direct match on tag
243  for (; it != list.end(); ++it)
244  {
245  if (it->GetCommand() == tag)
246  {
247  return *it;
248  }
249  }
250 
251  // no direct match. do we require a direct match? search for one that works
252  if (!absolute)
253  {
254  for (it = list.begin(); it != list.end(); ++it)
255  {
256  if (it->Accepts(tag))
257  {
258  return *it;
259  }
260  }
261  }
262 
263  // no working match. return a blank
264  return MetaGrabberScript();
265 }
266 
268  bool absolute)
269 {
270  static QRegExp retagref("^([a-zA-Z0-9_\\-\\.]+\\.[a-zA-Z0-9]{1,3})_(.*)");
271  static QRegExp retagref2("^([a-zA-Z0-9_\\-\\.]+\\.[a-zA-Z0-9]{1,3}):(.*)");
272  static QMutex reLock;
273  QMutexLocker lock(&reLock);
274  QString tag;
275 
276  if (retagref.indexIn(inetref) > -1)
277  {
278  tag = retagref.cap(1);
279  }
280  else if (retagref2.indexIn(inetref) > -1)
281  {
282  tag = retagref2.cap(1);
283  }
284  if (!tag.isEmpty())
285  {
286  // match found, pull out the grabber
287  MetaGrabberScript script = MetaGrabberScript::FromTag(tag, absolute);
288  if (script.IsValid())
289  return script;
290  }
291 
292  // no working match, return a blank
293  return MetaGrabberScript();
294 }
295 
296 QString MetaGrabberScript::CleanedInetref(const QString &inetref)
297 {
298  static QRegExp retagref("^([a-zA-Z0-9_\\-\\.]+\\.[a-zA-Z0-9]{1,3})_(.*)");
299  static QRegExp retagref2("^([a-zA-Z0-9_\\-\\.]+\\.[a-zA-Z0-9]{1,3}):(.*)");
300  static QMutex reLock;
301  QMutexLocker lock(&reLock);
302 
303  // try to strip grabber tag from inetref
304  if (retagref.indexIn(inetref) > -1)
305  return retagref.cap(2);
306  if (retagref2.indexIn(inetref) > -1)
307  return retagref2.cap(2);
308 
309  return inetref;
310 }
311 
313  const QDomElement &dom) :
314  m_fullcommand(path)
315 {
316  ParseGrabberVersion(dom);
317 }
318 
320 {
321  ParseGrabberVersion(dom);
322 }
323 
325 {
326  if (path.isEmpty())
327  return;
328  m_fullcommand = path;
329  if (path[0] != '/')
330  m_fullcommand.prepend(QString("%1metadata").arg(GetShareDir()));
331 
332  MythSystemLegacy grabber(path, QStringList() << "-v",
334  grabber.Run();
335  if (grabber.Wait() != GENERIC_EXIT_OK)
336  // script failed
337  return;
338 
339  QByteArray result = grabber.ReadAll();
340  if (result.isEmpty())
341  // no output
342  return;
343 
344  QDomDocument doc;
345  doc.setContent(result, true);
346  QDomElement root = doc.documentElement();
347  if (root.isNull())
348  // no valid XML
349  return;
350 
351  ParseGrabberVersion(root);
352  if (m_name.isEmpty())
353  // XML not processed correctly
354  return;
355 
356  m_valid = true;
357 }
358 
360 {
361  if (this != &other)
362  {
363  m_name = other.m_name;
364  m_author = other.m_author;
365  m_thumbnail = other.m_thumbnail;
366  m_command = other.m_command;
368  m_type = other.m_type;
369  m_typestring = other.m_typestring;
371  m_accepts = other.m_accepts;
372  m_version = other.m_version;
373  m_valid = other.m_valid;
374  }
375 
376  return *this;
377 }
378 
379 
380 void MetaGrabberScript::ParseGrabberVersion(const QDomElement &item)
381 {
382  m_name = item.firstChildElement("name").text();
383  m_author = item.firstChildElement("author").text();
384  m_thumbnail = item.firstChildElement("thumbnail").text();
385  m_command = item.firstChildElement("command").text();
386  m_description = item.firstChildElement("description").text();
387  m_version = item.firstChildElement("version").text().toFloat();
388  m_typestring = item.firstChildElement("type").text().toLower();
389 
390  if (!m_typestring.isEmpty() && grabberTypeStrings.contains(m_typestring))
392  else
394 
395  QDomElement accepts = item.firstChildElement("accepts");
396  if (!accepts.isNull())
397  {
398  while (!accepts.isNull())
399  {
400  m_accepts.append(accepts.text());
401  accepts = accepts.nextSiblingElement("accepts");
402  }
403  }
404 }
405 
407 {
408  if (!m_valid || m_fullcommand.isEmpty())
409  return false;
410 
411  QStringList args; args << "-t";
413 
414  grabber.Run();
415  return grabber.Wait() == GENERIC_EXIT_OK;
416 }
417 
418 // TODO
419 // using the MetadataLookup object as both argument input, and parsed output,
420 // is clumsy. break the inputs out into a separate object, and spawn a new
421 // MetadataLookup object in ParseMetadataItem, rather than requiring an
422 // existing one to reuse.
424  MetadataLookup *lookup, bool passseas)
425 {
427  MetadataLookupList list;
428 
429  LOG(VB_GENERAL, LOG_INFO, QString("Running Grabber: %1 %2")
430  .arg(m_fullcommand).arg(args.join(" ")));
431 
432  grabber.Run();
433  if (grabber.Wait(60) != GENERIC_EXIT_OK)
434  return list;
435 
436  QByteArray result = grabber.ReadAll();
437  if (!result.isEmpty())
438  {
439  QDomDocument doc;
440  doc.setContent(result, true);
441  QDomElement root = doc.documentElement();
442  QDomElement item = root.firstChildElement("item");
443 
444  while (!item.isNull())
445  {
446  MetadataLookup *tmp = ParseMetadataItem(item, lookup, passseas);
447  tmp->SetInetref(QString("%1_%2").arg(m_command)
448  .arg(tmp->GetInetref()));
449  if (!tmp->GetCollectionref().isEmpty())
450  {
451  tmp->SetCollectionref(QString("%1_%2").arg(m_command)
452  .arg(tmp->GetCollectionref()));
453  }
454  list.append(tmp);
455  // MetadataLookup is to be owned by the list
456  tmp->DecrRef();
457  item = item.nextSiblingElement("item");
458  }
459  }
460  return list;
461 }
462 
463 QString MetaGrabberScript::GetRelPath(void) const
464 {
465  QString share = GetShareDir();
466  if (m_fullcommand.startsWith(share))
467  return m_fullcommand.right(m_fullcommand.size() - share.size());
468 
469  return QString();
470 }
471 
472 void MetaGrabberScript::toMap(InfoMap &metadataMap) const
473 {
474  metadataMap["name"] = m_name;
475  metadataMap["author"] = m_author;
476  metadataMap["thumbnailfilename"] = m_thumbnail;
477  metadataMap["command"] = m_command;
478  metadataMap["description"] = m_description;
479  metadataMap["version"] = QString::number(m_version);
480  metadataMap["type"] = m_typestring;
481 }
482 
484 {
485  args << "-l"
487  << "-a"
489 }
490 
492  MetadataLookup *lookup, bool passseas)
493 {
494  QStringList args;
496 
497  args << "-M"
498  << title;
499 
500  return RunGrabber(args, lookup, passseas);
501 }
502 
504  const QString &subtitle, MetadataLookup *lookup,
505  bool passseas)
506 {
507  QStringList args;
509 
510  args << "-N"
511  << title
512  << subtitle;
513 
514  return RunGrabber(args, lookup, passseas);
515 }
516 
518  const QString &title, const QString &subtitle,
519  MetadataLookup *lookup, bool passseas)
520 {
521  (void)title;
522  QStringList args;
524 
525  args << "-N"
526  << CleanedInetref(inetref)
527  << subtitle;
528 
529  return RunGrabber(args, lookup, passseas);
530 }
531 
533  MetadataLookup *lookup, bool passseas)
534 {
535  QStringList args;
537 
538  args << "-D"
539  << CleanedInetref(inetref);
540 
541  return RunGrabber(args, lookup, passseas);
542 }
543 
545  int season, int episode, MetadataLookup *lookup,
546  bool passseas)
547 {
548  QStringList args;
550 
551  args << "-D"
552  << CleanedInetref(inetref)
553  << QString::number(season)
554  << QString::number(episode);
555 
556  return RunGrabber(args, lookup, passseas);
557 }
558 
560  const QString &collectionref, MetadataLookup *lookup,
561  bool passseas)
562 {
563  QStringList args;
565 
566  args << "-C"
567  << CleanedInetref(collectionref);
568 
569  return RunGrabber(args, lookup, passseas);
570 }
QStringList m_accepts
void Run(time_t timeout=0)
Runs a command inside the /bin/sh shell. Returns immediately.
MetadataLookup * ParseMetadataItem(const QDomElement &item, MetadataLookup *lookup, bool passseas)
QString GetRelPath(void) const
struct GrabberOpts GrabberOpts
MythLocale * GetLocale(void) const
allow access to stdout
Definition: mythsystem.h:39
#define LOC
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
static QMutex listLock
GrabberType m_type
static QString CleanedInetref(const QString &inetref)
static QDateTime grabberAge
GrabberType
MetadataLookupList Search(const QString &title, MetadataLookup *lookup, bool passseas=true)
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
static MetaGrabberScript FromTag(const QString &tag, bool absolute=false)
MetadataLookupList LookupCollection(const QString &collectionref, MetadataLookup *lookup, bool passseas=true)
static guint32 * tmp
Definition: goom_core.c:35
static GrabberOpts GrabberOptsMaker(QString thepath, QString thesetting, QString thedefault)
QString GetCountryCode() const
Definition: mythlocale.cpp:58
static GrabberList GetList(bool refresh=false)
static MetaGrabberScript GetGrabber(GrabberType defaultType, const MetadataLookup *lookup=nullptr)
MetadataLookupList RunGrabber(const QStringList &args, MetadataLookup *lookup, bool passseas)
MetadataLookupList SearchSubtitle(const QString &title, const QString &subtitle, MetadataLookup *lookup, bool passseas=true)
bool IsValid(void) const
QList< MetaGrabberScript > GrabberList
void ParseGrabberVersion(const QDomElement &item)
void toMap(InfoMap &metadataMap) const
#define kGrabberRefresh
QHash< QString, QString > InfoMap
Definition: mythtypes.h:15
run process through shell
Definition: mythsystem.h:41
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
QString GetSetting(const QString &key, const QString &defaultval="")
QByteArray & ReadAll()
QString GetShareDir(void)
Definition: mythdirs.cpp:222
void SetDefaultArgs(QStringList &args)
static QMap< QString, GrabberType > grabberTypeStrings
QString GetInetref() const
MetadataLookupList LookupData(const QString &inetref, MetadataLookup *lookup, bool passseas=true)
MetaGrabberScript()=default
static void InitializeStaticMaps(void)
uint Wait(time_t timeout=0)
static QMap< GrabberType, GrabberOpts > grabberTypes
static QMutex grabberLock
MetaGrabberScript & operator=(const MetaGrabberScript &other)
static bool initialized
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
static GrabberList grabberList
QString GetLanguage(void)
Returns two character ISO-639 language descriptor for UI language.
static MetaGrabberScript FromInetref(const QString &inetref, bool absolute=false)
GrabberType GetType(void) const
static QMutex typeLock