MythTV  master
metadatagrabber.cpp
Go to the documentation of this file.
1 // Qt headers
2 #include <QDateTime>
3 #include <QDir>
4 #include <QMap>
5 #include <QMutex>
6 #include <QMutexLocker>
7 #include <QRegExp>
8 #include <utility>
9 
10 // MythTV headers
11 #include "metadatagrabber.h"
12 #include "metadatacommon.h"
13 #include "mythsystemlegacy.h"
14 #include "exitcodes.h"
15 #include "mythdate.h"
16 #include "mythdirs.h"
17 #include "mythlogging.h"
18 #include "mythcorecontext.h"
19 
20 #define LOC QString("Metadata Grabber: ")
21 #define kGrabberRefresh 60
22 
24 static QMutex grabberLock;
25 static QDateTime grabberAge;
26 
27 typedef struct GrabberOpts {
28  QString path;
29  QString setting;
30  QString def;
31 } GrabberOpts;
32 
33 // TODO
34 // it would be nice to statically compile these, but I can't manage to get it
35 // to compile. apparently initializer lists are supported in QT5/CPP11 that
36 // will make this work. for now, use a lock and initialize on first access.
37 // https://bugreports.qt-project.org/browse/QTBUG-25679
38 static QMap<GrabberType, GrabberOpts> grabberTypes;
39 static QMap<QString, GrabberType> grabberTypeStrings;
40 static bool initialized = false;
41 static QMutex typeLock;
42 
43 static GrabberOpts GrabberOptsMaker(QString thepath, QString thesetting, QString thedefault)
44 {
46 
47  opts.path = std::move(thepath);
48  opts.setting = std::move(thesetting);
49  opts.def = std::move(thedefault);
50 
51  return opts;
52 }
53 
54 static void InitializeStaticMaps(void)
55 {
56  QMutexLocker lock(&typeLock);
57 
58  if (!initialized)
59  {
61  GrabberOptsMaker ("%1metadata/Movie/",
62  "MovieGrabber",
63  "metadata/Movie/tmdb3.py" );
65  GrabberOptsMaker ( "%1metadata/Television/",
66  "TelevisionGrabber",
67  "metadata/Television/ttvdb.py" );
69  GrabberOptsMaker ( "%1metadata/Game/",
70  "mythgame.MetadataGrabber",
71  "metadata/Game/giantbomb.py" );
73  GrabberOptsMaker ( "%1metadata/Music",
74  "",
75  "" );
76 
78  grabberTypeStrings["television"] = kGrabberTelevision;
81 
82 
83  initialized = true;
84  }
85 }
86 
88 {
89  return MetaGrabberScript::GetList(kGrabberAll, refresh);
90 }
91 
92 GrabberList MetaGrabberScript::GetList(const QString &type, bool refresh)
93 {
94  QString tmptype = type.toLower();
95  if (!grabberTypeStrings.contains(tmptype))
96  // unknown type, return empty list
97  return GrabberList();
98 
99  return MetaGrabberScript::GetList(grabberTypeStrings[tmptype], refresh);
100 }
101 
103  bool refresh)
104 {
106 
107  GrabberList tmpGrabberList, retGrabberList;
108  {
109  QMutexLocker listLock(&grabberLock);
110  QDateTime now = MythDate::current();
111 
112  // refresh grabber scripts every 60 seconds
113  // this might have to be revised, or made more intelligent if
114  // the delay during refreshes is too great
115  if (refresh || !grabberAge.isValid() ||
116  (grabberAge.secsTo(now) > kGrabberRefresh))
117  {
118  grabberList.clear();
119  LOG(VB_GENERAL, LOG_DEBUG, LOC + "Clearing grabber cache");
120 
121  // loop through different types of grabber scripts and the
122  // directories they are stored in
123  QMap<GrabberType, GrabberOpts>::const_iterator it;
124  for (it = grabberTypes.begin(); it != grabberTypes.end(); ++it)
125  {
126  QString path = (it->path).arg(GetShareDir());
127  QStringList scripts = QDir(path).entryList(QDir::Executable | QDir::Files);
128  if (scripts.count() == 0)
129  // no scripts found
130  continue;
131 
132  // loop through discovered scripts
133  QStringList::const_iterator it2 = scripts.begin();
134  for (; it2 != scripts.end(); ++it2)
135  {
136  QString cmd = QDir(path).filePath(*it2);
137  MetaGrabberScript script(cmd);
138 
139  if (script.IsValid())
140  {
141  LOG(VB_GENERAL, LOG_DEBUG, LOC + "Adding " + script.m_command);
142  grabberList.append(script);
143  }
144  }
145  }
146 
147  grabberAge = now;
148  }
149 
150  tmpGrabberList = grabberList;
151  }
152 
153  GrabberList::const_iterator it = tmpGrabberList.begin();
154  for (; it != tmpGrabberList.end(); ++it)
155  {
156  if ((type == kGrabberAll) || (it->GetType() == type))
157  retGrabberList.append(*it);
158  }
159 
160  return retGrabberList;
161 }
162 
164  const MetadataLookup *lookup)
165 {
166  if (!lookup)
167  {
168  return GetType(defaultType);
169  }
170 
171  if (!lookup->GetInetref().isEmpty() &&
172  lookup->GetInetref() != "00000000")
173  {
174  // inetref is defined, see if we have a pre-defined grabber
175  MetaGrabberScript grabber = FromInetref(lookup->GetInetref());
176 
177  if (grabber.IsValid())
178  {
179  return grabber;
180  }
181  // matching grabber was not found, just use the default
182  // fall through
183  }
184 
185  return GetType(defaultType);
186 }
187 
189 {
190  QString tmptype = type.toLower();
191  if (!grabberTypeStrings.contains(tmptype))
192  // unknown type, return empty grabber
193  return MetaGrabberScript();
194 
196 }
197 
199 {
201 
202  QString cmd = gCoreContext->GetSetting(grabberTypes[type].setting,
203  grabberTypes[type].def);
204 
205  if (cmd.isEmpty())
206  {
207  // should the python bindings had not been installed at any stage
208  // the settings could have been set to an empty string, so use default
209  cmd = grabberTypes[type].def;
210  }
211 
212  if (grabberAge.isValid() && grabberAge.secsTo(MythDate::current()) <= kGrabberRefresh)
213  {
214  // just pull it from the cache
215  GrabberList list = GetList();
216  GrabberList::const_iterator it = list.begin();
217 
218  for (; it != list.end(); ++it)
219  if (it->GetPath().endsWith(cmd))
220  return *it;
221  }
222 
223  // polling the cache will cause a refresh, so lets just grab and
224  // process the script directly
225  QString fullcmd = QString("%1%2").arg(GetShareDir()).arg(cmd);
226  MetaGrabberScript script(fullcmd);
227 
228  if (script.IsValid())
229  {
230  return script;
231  }
232 
233  return MetaGrabberScript();
234 }
235 
237  bool absolute)
238 {
239  GrabberList list = GetList();
240  GrabberList::const_iterator it = list.begin();
241 
242  // search for direct match on tag
243  for (; it != list.end(); ++it)
244  {
245  if (it->GetCommand() == tag)
246  {
247  return *it;
248  }
249  }
250 
251  // no direct match. do we require a direct match? search for one that works
252  if (!absolute)
253  {
254  for (it = list.begin(); it != list.end(); ++it)
255  {
256  if (it->Accepts(tag))
257  {
258  return *it;
259  }
260  }
261  }
262 
263  // no working match. return a blank
264  return MetaGrabberScript();
265 }
266 
268  bool absolute)
269 {
270  static QRegExp retagref("^([a-zA-Z0-9_\\-\\.]+\\.[a-zA-Z0-9]{1,3})_(.*)");
271  static QRegExp retagref2("^([a-zA-Z0-9_\\-\\.]+\\.[a-zA-Z0-9]{1,3}):(.*)");
272  static QMutex reLock;
273  QMutexLocker lock(&reLock);
274  QString tag;
275 
276  if (retagref.indexIn(inetref) > -1)
277  {
278  tag = retagref.cap(1);
279  }
280  else if (retagref2.indexIn(inetref) > -1)
281  {
282  tag = retagref2.cap(1);
283  }
284  if (!tag.isEmpty())
285  {
286  // match found, pull out the grabber
287  MetaGrabberScript script = MetaGrabberScript::FromTag(tag, absolute);
288  if (script.IsValid())
289  return script;
290  }
291 
292  // no working match, return a blank
293  return MetaGrabberScript();
294 }
295 
296 QString MetaGrabberScript::CleanedInetref(const QString &inetref)
297 {
298  static QRegExp retagref("^([a-zA-Z0-9_\\-\\.]+\\.[a-zA-Z0-9]{1,3})_(.*)");
299  static QRegExp retagref2("^([a-zA-Z0-9_\\-\\.]+\\.[a-zA-Z0-9]{1,3}):(.*)");
300  static QMutex reLock;
301  QMutexLocker lock(&reLock);
302 
303  // try to strip grabber tag from inetref
304  if (retagref.indexIn(inetref) > -1)
305  return retagref.cap(2);
306  if (retagref2.indexIn(inetref) > -1)
307  return retagref2.cap(2);
308 
309  return inetref;
310 }
311 
312 MetaGrabberScript::MetaGrabberScript(QString path, const QDomElement &dom) :
313  m_fullcommand(std::move(path))
314 {
315  ParseGrabberVersion(dom);
316 }
317 
319 {
320  ParseGrabberVersion(dom);
321 }
322 
324 {
325  if (path.isEmpty())
326  return;
327  m_fullcommand = path;
328  if (path[0] != '/')
329  m_fullcommand.prepend(QString("%1metadata").arg(GetShareDir()));
330 
331  MythSystemLegacy grabber(path, QStringList() << "-v",
333  grabber.Run();
334  if (grabber.Wait() != GENERIC_EXIT_OK)
335  // script failed
336  return;
337 
338  QByteArray result = grabber.ReadAll();
339  if (result.isEmpty())
340  // no output
341  return;
342 
343  QDomDocument doc;
344  doc.setContent(result, true);
345  QDomElement root = doc.documentElement();
346  if (root.isNull())
347  // no valid XML
348  return;
349 
350  ParseGrabberVersion(root);
351  if (m_name.isEmpty())
352  // XML not processed correctly
353  return;
354 
355  m_valid = true;
356 }
357 
359 {
360  if (this != &other)
361  {
362  m_name = other.m_name;
363  m_author = other.m_author;
364  m_thumbnail = other.m_thumbnail;
365  m_command = other.m_command;
367  m_type = other.m_type;
368  m_typestring = other.m_typestring;
370  m_accepts = other.m_accepts;
371  m_version = other.m_version;
372  m_valid = other.m_valid;
373  }
374 
375  return *this;
376 }
377 
378 
379 void MetaGrabberScript::ParseGrabberVersion(const QDomElement &item)
380 {
381  m_name = item.firstChildElement("name").text();
382  m_author = item.firstChildElement("author").text();
383  m_thumbnail = item.firstChildElement("thumbnail").text();
384  m_command = item.firstChildElement("command").text();
385  m_description = item.firstChildElement("description").text();
386  m_version = item.firstChildElement("version").text().toFloat();
387  m_typestring = item.firstChildElement("type").text().toLower();
388 
389  if (!m_typestring.isEmpty() && grabberTypeStrings.contains(m_typestring))
391  else
393 
394  QDomElement accepts = item.firstChildElement("accepts");
395  if (!accepts.isNull())
396  {
397  while (!accepts.isNull())
398  {
399  m_accepts.append(accepts.text());
400  accepts = accepts.nextSiblingElement("accepts");
401  }
402  }
403 }
404 
406 {
407  if (!m_valid || m_fullcommand.isEmpty())
408  return false;
409 
410  QStringList args; args << "-t";
412 
413  grabber.Run();
414  return grabber.Wait() == GENERIC_EXIT_OK;
415 }
416 
417 // TODO
418 // using the MetadataLookup object as both argument input, and parsed output,
419 // is clumsy. break the inputs out into a separate object, and spawn a new
420 // MetadataLookup object in ParseMetadataItem, rather than requiring an
421 // existing one to reuse.
423  MetadataLookup *lookup, bool passseas)
424 {
426  MetadataLookupList list;
427 
428  LOG(VB_GENERAL, LOG_INFO, QString("Running Grabber: %1 %2")
429  .arg(m_fullcommand).arg(args.join(" ")));
430 
431  grabber.Run();
432  if (grabber.Wait(60) != GENERIC_EXIT_OK)
433  return list;
434 
435  QByteArray result = grabber.ReadAll();
436  if (!result.isEmpty())
437  {
438  QDomDocument doc;
439  doc.setContent(result, true);
440  QDomElement root = doc.documentElement();
441  QDomElement item = root.firstChildElement("item");
442 
443  while (!item.isNull())
444  {
445  MetadataLookup *tmp = ParseMetadataItem(item, lookup, passseas);
446  tmp->SetInetref(QString("%1_%2").arg(m_command)
447  .arg(tmp->GetInetref()));
448  if (!tmp->GetCollectionref().isEmpty())
449  {
450  tmp->SetCollectionref(QString("%1_%2").arg(m_command)
451  .arg(tmp->GetCollectionref()));
452  }
453  list.append(tmp);
454  // MetadataLookup is to be owned by the list
455  tmp->DecrRef();
456  item = item.nextSiblingElement("item");
457  }
458  }
459  return list;
460 }
461 
462 QString MetaGrabberScript::GetRelPath(void) const
463 {
464  QString share = GetShareDir();
465  if (m_fullcommand.startsWith(share))
466  return m_fullcommand.right(m_fullcommand.size() - share.size());
467 
468  return QString();
469 }
470 
471 void MetaGrabberScript::toMap(InfoMap &metadataMap) const
472 {
473  metadataMap["name"] = m_name;
474  metadataMap["author"] = m_author;
475  metadataMap["thumbnailfilename"] = m_thumbnail;
476  metadataMap["command"] = m_command;
477  metadataMap["description"] = m_description;
478  metadataMap["version"] = QString::number(m_version);
479  metadataMap["type"] = m_typestring;
480 }
481 
483 {
484  args << "-l"
486  << "-a"
488 }
489 
491  MetadataLookup *lookup, bool passseas)
492 {
493  QStringList args;
495 
496  args << "-M"
497  << title;
498 
499  return RunGrabber(args, lookup, passseas);
500 }
501 
503  const QString &subtitle, MetadataLookup *lookup,
504  bool passseas)
505 {
506  QStringList args;
508 
509  args << "-N"
510  << title
511  << subtitle;
512 
513  return RunGrabber(args, lookup, passseas);
514 }
515 
517  const QString &title, const QString &subtitle,
518  MetadataLookup *lookup, bool passseas)
519 {
520  (void)title;
521  QStringList args;
523 
524  args << "-N"
525  << CleanedInetref(inetref)
526  << subtitle;
527 
528  return RunGrabber(args, lookup, passseas);
529 }
530 
532  MetadataLookup *lookup, bool passseas)
533 {
534  QStringList args;
536 
537  args << "-D"
538  << CleanedInetref(inetref);
539 
540  return RunGrabber(args, lookup, passseas);
541 }
542 
544  int season, int episode, MetadataLookup *lookup,
545  bool passseas)
546 {
547  QStringList args;
549 
550  args << "-D"
551  << CleanedInetref(inetref)
552  << QString::number(season)
553  << QString::number(episode);
554 
555  return RunGrabber(args, lookup, passseas);
556 }
557 
559  const QString &collectionref, MetadataLookup *lookup,
560  bool passseas)
561 {
562  QStringList args;
564 
565  args << "-C"
566  << CleanedInetref(collectionref);
567 
568  return RunGrabber(args, lookup, passseas);
569 }
QStringList m_accepts
void Run(time_t timeout=0)
Runs a command inside the /bin/sh shell. Returns immediately.
MetadataLookup * ParseMetadataItem(const QDomElement &item, MetadataLookup *lookup, bool passseas)
QString GetRelPath(void) const
struct GrabberOpts GrabberOpts
MythLocale * GetLocale(void) const
allow access to stdout
Definition: mythsystem.h:39
#define LOC
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
static QMutex listLock
GrabberType m_type
static QString CleanedInetref(const QString &inetref)
static QDateTime grabberAge
GrabberType
MetadataLookupList Search(const QString &title, MetadataLookup *lookup, bool passseas=true)
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
static MetaGrabberScript FromTag(const QString &tag, bool absolute=false)
MetadataLookupList LookupCollection(const QString &collectionref, MetadataLookup *lookup, bool passseas=true)
static guint32 * tmp
Definition: goom_core.c:35
static GrabberOpts GrabberOptsMaker(QString thepath, QString thesetting, QString thedefault)
QString GetCountryCode() const
Definition: mythlocale.cpp:58
static GrabberList GetList(bool refresh=false)
static MetaGrabberScript GetGrabber(GrabberType defaultType, const MetadataLookup *lookup=nullptr)
MetadataLookupList RunGrabber(const QStringList &args, MetadataLookup *lookup, bool passseas)
MetadataLookupList SearchSubtitle(const QString &title, const QString &subtitle, MetadataLookup *lookup, bool passseas=true)
bool IsValid(void) const
QList< MetaGrabberScript > GrabberList
void ParseGrabberVersion(const QDomElement &item)
void toMap(InfoMap &metadataMap) const
#define kGrabberRefresh
QHash< QString, QString > InfoMap
Definition: mythtypes.h:15
run process through shell
Definition: mythsystem.h:41
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
QString GetSetting(const QString &key, const QString &defaultval="")
QByteArray & ReadAll()
QString GetShareDir(void)
Definition: mythdirs.cpp:222
static void SetDefaultArgs(QStringList &args)
static QMap< QString, GrabberType > grabberTypeStrings
QString GetInetref() const
MetadataLookupList LookupData(const QString &inetref, MetadataLookup *lookup, bool passseas=true)
MetaGrabberScript()=default
static void InitializeStaticMaps(void)
uint Wait(time_t timeout=0)
static QMap< GrabberType, GrabberOpts > grabberTypes
static QMutex grabberLock
MetaGrabberScript & operator=(const MetaGrabberScript &other)
static bool initialized
#define LOG(_MASK_, _LEVEL_, _STRING_)
Definition: mythlogging.h:41
static GrabberList grabberList
QString GetLanguage(void)
Returns two character ISO-639 language descriptor for UI language.
static MetaGrabberScript FromInetref(const QString &inetref, bool absolute=false)
GrabberType GetType(void) const
static QMutex typeLock