MythTV  master
metadatagrabber.cpp
Go to the documentation of this file.
1 // Qt headers
2 #include <QDateTime>
3 #include <QDir>
4 #include <QMap>
5 #include <QMutex>
6 #include <QMutexLocker>
7 #include <QRegExp>
8 #include <utility>
9 
10 // MythTV headers
11 #include "metadatagrabber.h"
12 #include "metadatacommon.h"
13 #include "mythsystemlegacy.h"
14 #include "exitcodes.h"
15 #include "mythdate.h"
16 #include "mythdirs.h"
17 #include "mythlogging.h"
18 #include "mythcorecontext.h"
19 
20 #define LOC QString("Metadata Grabber: ")
21 #define kGrabberRefresh 60
22 
24 static QMutex grabberLock;
25 static QDateTime grabberAge;
26 
27 struct GrabberOpts {
28  QString m_path;
29  QString m_setting;
30  QString m_def;
31 };
32 
33 static const QMap<GrabberType, GrabberOpts> grabberTypes {
34  { kGrabberMovie, { "%1metadata/Movie/",
35  "MovieGrabber",
36  "metadata/Movie/tmdb3.py" } },
37  { kGrabberTelevision, { "%1metadata/Television/",
38  "TelevisionGrabber",
39  "metadata/Television/ttvdb.py" } },
40  { kGrabberGame, { "%1metadata/Game/",
41  "mythgame.MetadataGrabber",
42  "metadata/Game/giantbomb.py" } },
43  { kGrabberMusic, { "%1metadata/Music",
44  "",
45  "" } }
46 };
47 
48 static QMap<QString, GrabberType> grabberTypeStrings {
49  { "movie", kGrabberMovie },
50  { "television", kGrabberTelevision },
51  { "game", kGrabberGame },
52  { "music", kGrabberMusic }
53 };
54 
56 {
57  return MetaGrabberScript::GetList(kGrabberAll, refresh);
58 }
59 
60 GrabberList MetaGrabberScript::GetList(const QString &type, bool refresh)
61 {
62  QString tmptype = type.toLower();
63  if (!grabberTypeStrings.contains(tmptype))
64  // unknown type, return empty list
65  return GrabberList();
66 
67  return MetaGrabberScript::GetList(grabberTypeStrings[tmptype], refresh);
68 }
69 
71  bool refresh)
72 {
73  GrabberList tmpGrabberList;
74  GrabberList retGrabberList;
75  {
76  QMutexLocker listLock(&grabberLock);
77  QDateTime now = MythDate::current();
78 
79  // refresh grabber scripts every 60 seconds
80  // this might have to be revised, or made more intelligent if
81  // the delay during refreshes is too great
82  if (refresh || !grabberAge.isValid() ||
83  (grabberAge.secsTo(now) > kGrabberRefresh))
84  {
85  grabberList.clear();
86  LOG(VB_GENERAL, LOG_DEBUG, LOC + "Clearing grabber cache");
87 
88  // loop through different types of grabber scripts and the
89  // directories they are stored in
90  for (const auto& grabberType : qAsConst(grabberTypes))
91  {
92  QString path = (grabberType.m_path).arg(GetShareDir());
93  QStringList scripts = QDir(path).entryList(QDir::Executable | QDir::Files);
94  if (scripts.count() == 0)
95  // no scripts found
96  continue;
97 
98  // loop through discovered scripts
99  for (const auto& name : qAsConst(scripts))
100  {
101  QString cmd = QDir(path).filePath(name);
102  MetaGrabberScript script(cmd);
103 
104  if (script.IsValid())
105  {
106  LOG(VB_GENERAL, LOG_DEBUG, LOC + "Adding " + script.m_command);
107  grabberList.append(script);
108  }
109  }
110  }
111 
112  grabberAge = now;
113  }
114 
115  tmpGrabberList = grabberList;
116  }
117 
118  for (const auto& item : qAsConst(tmpGrabberList))
119  {
120  if ((type == kGrabberAll) || (item.GetType() == type))
121  retGrabberList.append(item);
122  }
123 
124  return retGrabberList;
125 }
126 
128  const MetadataLookup *lookup)
129 {
130  if (!lookup)
131  {
132  return GetType(defaultType);
133  }
134 
135  if (!lookup->GetInetref().isEmpty() &&
136  lookup->GetInetref() != "00000000")
137  {
138  // inetref is defined, see if we have a pre-defined grabber
139  MetaGrabberScript grabber = FromInetref(lookup->GetInetref());
140 
141  if (grabber.IsValid())
142  {
143  return grabber;
144  }
145  // matching grabber was not found, just use the default
146  // fall through
147  }
148 
149  return GetType(defaultType);
150 }
151 
153 {
154  QString tmptype = type.toLower();
155  if (!grabberTypeStrings.contains(tmptype))
156  // unknown type, return empty grabber
157  return MetaGrabberScript();
158 
160 }
161 
163 {
164  QString cmd = gCoreContext->GetSetting(grabberTypes[type].m_setting,
165  grabberTypes[type].m_def);
166 
167  if (cmd.isEmpty())
168  {
169  // should the python bindings had not been installed at any stage
170  // the settings could have been set to an empty string, so use default
171  cmd = grabberTypes[type].m_def;
172  }
173 
174  if (grabberAge.isValid() && grabberAge.secsTo(MythDate::current()) <= kGrabberRefresh)
175  {
176  // just pull it from the cache
177  GrabberList list = GetList();
178  for (const auto& item : qAsConst(list))
179  if (item.GetPath().endsWith(cmd))
180  return item;
181  }
182 
183  // polling the cache will cause a refresh, so lets just grab and
184  // process the script directly
185  QString fullcmd = QString("%1%2").arg(GetShareDir()).arg(cmd);
186  MetaGrabberScript script(fullcmd);
187 
188  if (script.IsValid())
189  {
190  return script;
191  }
192 
193  return MetaGrabberScript();
194 }
195 
197  bool absolute)
198 {
199  GrabberList list = GetList();
200 
201  // search for direct match on tag
202  for (const auto& item : qAsConst(list))
203  {
204  if (item.GetCommand() == tag)
205  {
206  return item;
207  }
208  }
209 
210  // no direct match. do we require a direct match? search for one that works
211  if (!absolute)
212  {
213  for (const auto& item : qAsConst(list))
214  {
215  if (item.Accepts(tag))
216  {
217  return item;
218  }
219  }
220  }
221 
222  // no working match. return a blank
223  return MetaGrabberScript();
224 }
225 
227  bool absolute)
228 {
229  static QRegExp s_retagref(R"(^([a-zA-Z0-9_\-\.]+\.[a-zA-Z0-9]{1,3})_(.*))");
230  static QRegExp s_retagref2(R"(^([a-zA-Z0-9_\-\.]+\.[a-zA-Z0-9]{1,3}):(.*))");
231  static QMutex s_reLock;
232  QMutexLocker lock(&s_reLock);
233  QString tag;
234 
235  if (s_retagref.indexIn(inetref) > -1)
236  {
237  tag = s_retagref.cap(1);
238  }
239  else if (s_retagref2.indexIn(inetref) > -1)
240  {
241  tag = s_retagref2.cap(1);
242  }
243  if (!tag.isEmpty())
244  {
245  // match found, pull out the grabber
246  MetaGrabberScript script = MetaGrabberScript::FromTag(tag, absolute);
247  if (script.IsValid())
248  return script;
249  }
250 
251  // no working match, return a blank
252  return MetaGrabberScript();
253 }
254 
255 QString MetaGrabberScript::CleanedInetref(const QString &inetref)
256 {
257  static QRegExp s_retagref(R"(^([a-zA-Z0-9_\-\.]+\.[a-zA-Z0-9]{1,3})_(.*))");
258  static QRegExp s_retagref2(R"(^([a-zA-Z0-9_\-\.]+\.[a-zA-Z0-9]{1,3}):(.*))");
259  static QMutex s_reLock;
260  QMutexLocker lock(&s_reLock);
261 
262  // try to strip grabber tag from inetref
263  if (s_retagref.indexIn(inetref) > -1)
264  return s_retagref.cap(2);
265  if (s_retagref2.indexIn(inetref) > -1)
266  return s_retagref2.cap(2);
267 
268  return inetref;
269 }
270 
271 MetaGrabberScript::MetaGrabberScript(QString path, const QDomElement &dom) :
272  m_fullcommand(std::move(path))
273 {
274  ParseGrabberVersion(dom);
275 }
276 
278 {
279  ParseGrabberVersion(dom);
280 }
281 
283 {
284  if (path.isEmpty())
285  return;
286  m_fullcommand = path;
287  if (path[0] != '/')
288  m_fullcommand.prepend(QString("%1metadata").arg(GetShareDir()));
289 
290  MythSystemLegacy grabber(path, QStringList() << "-v",
292  grabber.Run();
293  if (grabber.Wait() != GENERIC_EXIT_OK)
294  // script failed
295  return;
296 
297  QByteArray result = grabber.ReadAll();
298  if (result.isEmpty())
299  // no output
300  return;
301 
302  QDomDocument doc;
303  doc.setContent(result, true);
304  QDomElement root = doc.documentElement();
305  if (root.isNull())
306  // no valid XML
307  return;
308 
310  if (m_name.isEmpty())
311  // XML not processed correctly
312  return;
313 
314  m_valid = true;
315 }
316 
318 {
319  if (this != &other)
320  {
321  m_name = other.m_name;
322  m_author = other.m_author;
323  m_thumbnail = other.m_thumbnail;
324  m_command = other.m_command;
326  m_type = other.m_type;
327  m_typestring = other.m_typestring;
329  m_accepts = other.m_accepts;
330  m_version = other.m_version;
331  m_valid = other.m_valid;
332  }
333 
334  return *this;
335 }
336 
337 
338 void MetaGrabberScript::ParseGrabberVersion(const QDomElement &item)
339 {
340  m_name = item.firstChildElement("name").text();
341  m_author = item.firstChildElement("author").text();
342  m_thumbnail = item.firstChildElement("thumbnail").text();
343  m_command = item.firstChildElement("command").text();
344  m_description = item.firstChildElement("description").text();
345  m_version = item.firstChildElement("version").text().toFloat();
346  m_typestring = item.firstChildElement("type").text().toLower();
347 
348  if (!m_typestring.isEmpty() && grabberTypeStrings.contains(m_typestring))
350  else
352 
353  QDomElement accepts = item.firstChildElement("accepts");
354  if (!accepts.isNull())
355  {
356  while (!accepts.isNull())
357  {
358  m_accepts.append(accepts.text());
359  accepts = accepts.nextSiblingElement("accepts");
360  }
361  }
362 }
363 
365 {
366  if (!m_valid || m_fullcommand.isEmpty())
367  return false;
368 
369  QStringList args; args << "-t";
371 
372  grabber.Run();
373  return grabber.Wait() == GENERIC_EXIT_OK;
374 }
375 
376 // TODO
377 // using the MetadataLookup object as both argument input, and parsed output,
378 // is clumsy. break the inputs out into a separate object, and spawn a new
379 // MetadataLookup object in ParseMetadataItem, rather than requiring an
380 // existing one to reuse.
382  MetadataLookup *lookup, bool passseas)
383 {
385  MetadataLookupList list;
386 
387  LOG(VB_GENERAL, LOG_INFO, QString("Running Grabber: %1 %2")
388  .arg(m_fullcommand).arg(args.join(" ")));
389 
390  grabber.Run();
391  if (grabber.Wait(60) != GENERIC_EXIT_OK)
392  return list;
393 
394  QByteArray result = grabber.ReadAll();
395  if (!result.isEmpty())
396  {
397  QDomDocument doc;
398  doc.setContent(result, true);
399  QDomElement root = doc.documentElement();
400  QDomElement item = root.firstChildElement("item");
401 
402  while (!item.isNull())
403  {
404  MetadataLookup *tmp = ParseMetadataItem(item, lookup, passseas);
405  tmp->SetInetref(QString("%1_%2").arg(m_command)
406  .arg(tmp->GetInetref()));
407  if (!tmp->GetCollectionref().isEmpty())
408  {
409  tmp->SetCollectionref(QString("%1_%2").arg(m_command)
410  .arg(tmp->GetCollectionref()));
411  }
412  list.append(tmp);
413  // MetadataLookup is to be owned by the list
414  tmp->DecrRef();
415  item = item.nextSiblingElement("item");
416  }
417  }
418  return list;
419 }
420 
421 QString MetaGrabberScript::GetRelPath(void) const
422 {
423  QString share = GetShareDir();
424  if (m_fullcommand.startsWith(share))
425  return m_fullcommand.right(m_fullcommand.size() - share.size());
426 
427  return QString();
428 }
429 
430 void MetaGrabberScript::toMap(InfoMap &metadataMap) const
431 {
432  metadataMap["name"] = m_name;
433  metadataMap["author"] = m_author;
434  metadataMap["thumbnailfilename"] = m_thumbnail;
435  metadataMap["command"] = m_command;
436  metadataMap["description"] = m_description;
437  metadataMap["version"] = QString::number(m_version);
438  metadataMap["type"] = m_typestring;
439 }
440 
442 {
443  args << "-l"
445  << "-a"
447 }
448 
450  MetadataLookup *lookup, bool passseas)
451 {
452  QStringList args;
454 
455  args << "-M"
456  << title;
457 
458  return RunGrabber(args, lookup, passseas);
459 }
460 
462  const QString &subtitle, MetadataLookup *lookup,
463  bool passseas)
464 {
465  QStringList args;
467 
468  args << "-N"
469  << title
470  << subtitle;
471 
472  return RunGrabber(args, lookup, passseas);
473 }
474 
476  const QString &title, const QString &subtitle,
477  MetadataLookup *lookup, bool passseas)
478 {
479  (void)title;
480  QStringList args;
482 
483  args << "-N"
484  << CleanedInetref(inetref)
485  << subtitle;
486 
487  return RunGrabber(args, lookup, passseas);
488 }
489 
491  MetadataLookup *lookup, bool passseas)
492 {
493  QStringList args;
495 
496  args << "-D"
497  << CleanedInetref(inetref);
498 
499  return RunGrabber(args, lookup, passseas);
500 }
501 
503  int season, int episode, MetadataLookup *lookup,
504  bool passseas)
505 {
506  QStringList args;
508 
509  args << "-D"
510  << CleanedInetref(inetref)
511  << QString::number(season)
512  << QString::number(episode);
513 
514  return RunGrabber(args, lookup, passseas);
515 }
516 
518  const QString &collectionref, MetadataLookup *lookup,
519  bool passseas)
520 {
521  QStringList args;
523 
524  args << "-C"
525  << CleanedInetref(collectionref);
526 
527  return RunGrabber(args, lookup, passseas);
528 }
LOC
#define LOC
Definition: metadatagrabber.cpp:20
build_compdb.args
args
Definition: build_compdb.py:11
grabberLock
static QMutex grabberLock
Definition: metadatagrabber.cpp:24
MetaGrabberScript
Definition: metadatagrabber.h:30
MetaGrabberScript::m_fullcommand
QString m_fullcommand
Definition: metadatagrabber.h:84
GrabberType
GrabberType
Definition: metadatagrabber.h:20
MetaGrabberScript::LookupCollection
MetadataLookupList LookupCollection(const QString &collectionref, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:517
GENERIC_EXIT_OK
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
kGrabberTelevision
@ kGrabberTelevision
Definition: metadatagrabber.h:23
root
QDomElement root
Definition: mythplugins/mytharchive/mytharchivehelper/main.cpp:653
GrabberList
QList< MetaGrabberScript > GrabberList
Definition: metadatagrabber.h:18
MetaGrabberScript::GetList
static GrabberList GetList(bool refresh=false)
Definition: metadatagrabber.cpp:55
MythSystemLegacy
Definition: mythsystemlegacy.h:69
GrabberOpts::m_setting
QString m_setting
Definition: metadatagrabber.cpp:29
doc
QDomDocument doc("MYTHARCHIVEITEM")
MetaGrabberScript::Test
bool Test(void)
Definition: metadatagrabber.cpp:364
MetaGrabberScript::SearchSubtitle
MetadataLookupList SearchSubtitle(const QString &title, const QString &subtitle, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:461
title
QString title
Definition: mythplugins/mytharchive/mytharchivehelper/main.cpp:633
MythCoreContext::GetLocale
MythLocale * GetLocale(void) const
Definition: mythcorecontext.cpp:1772
MetaGrabberScript::ParseGrabberVersion
void ParseGrabberVersion(const QDomElement &item)
Definition: metadatagrabber.cpp:338
MetaGrabberScript::m_accepts
QStringList m_accepts
Definition: metadatagrabber.h:89
arg
arg(title).arg(filename).arg(doDelete))
MetaGrabberScript::GetRelPath
QString GetRelPath(void) const
Definition: metadatagrabber.cpp:421
MetaGrabberScript::IsValid
bool IsValid(void) const
Definition: metadatagrabber.h:55
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23
MythSystemLegacy::ReadAll
QByteArray & ReadAll()
Definition: mythsystemlegacy.cpp:397
MetaGrabberScript::toMap
void toMap(InfoMap &metadataMap) const
Definition: metadatagrabber.cpp:430
MetaGrabberScript::m_type
GrabberType m_type
Definition: metadatagrabber.h:86
MythSystemLegacy::Run
void Run(time_t timeout=0)
Runs a command inside the /bin/sh shell. Returns immediately.
Definition: mythsystemlegacy.cpp:212
mythdirs.h
MetaGrabberScript::m_thumbnail
QString m_thumbnail
Definition: metadatagrabber.h:83
MetaGrabberScript::SetDefaultArgs
static void SetDefaultArgs(QStringList &args)
Definition: metadatagrabber.cpp:441
kGrabberRefresh
#define kGrabberRefresh
Definition: metadatagrabber.cpp:21
grabberAge
static QDateTime grabberAge
Definition: metadatagrabber.cpp:25
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
metadatagrabber.h
tmp
static guint32 * tmp
Definition: goom_core.cpp:30
kGrabberMovie
@ kGrabberMovie
Definition: metadatagrabber.h:22
GrabberOpts::m_path
QString m_path
Definition: metadatagrabber.cpp:28
mythsystemlegacy.h
MetadataLookup
Definition: metadatacommon.h:87
InfoMap
QHash< QString, QString > InfoMap
Definition: mythtypes.h:15
kGrabberMusic
@ kGrabberMusic
Definition: metadatagrabber.h:24
grabberList
static GrabberList grabberList
Definition: metadatagrabber.cpp:23
MetaGrabberScript::CleanedInetref
static QString CleanedInetref(const QString &inetref)
Definition: metadatagrabber.cpp:255
mythdate.h
MetaGrabberScript::Search
MetadataLookupList Search(const QString &title, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:449
mythlogging.h
MythLocale::GetCountryCode
QString GetCountryCode() const
Definition: mythlocale.cpp:58
MetaGrabberScript::m_command
QString m_command
Definition: metadatagrabber.h:85
RefCountedList< MetadataLookup >
GrabberOpts::m_def
QString m_def
Definition: metadatagrabber.cpp:30
GetShareDir
QString GetShareDir(void)
Definition: mythdirs.cpp:222
MetaGrabberScript::m_name
QString m_name
Definition: metadatagrabber.h:81
MetaGrabberScript::FromTag
static MetaGrabberScript FromTag(const QString &tag, bool absolute=false)
Definition: metadatagrabber.cpp:196
MetaGrabberScript::operator=
MetaGrabberScript & operator=(const MetaGrabberScript &other)
Definition: metadatagrabber.cpp:317
MetaGrabberScript::m_author
QString m_author
Definition: metadatagrabber.h:82
MetaGrabberScript::m_description
QString m_description
Definition: metadatagrabber.h:88
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:57
MetaGrabberScript::RunGrabber
MetadataLookupList RunGrabber(const QStringList &args, MetadataLookup *lookup, bool passseas)
Definition: metadatagrabber.cpp:381
MetaGrabberScript::GetGrabber
static MetaGrabberScript GetGrabber(GrabberType defaultType, const MetadataLookup *lookup=nullptr)
Definition: metadatagrabber.cpp:127
MythCoreContext::GetLanguage
QString GetLanguage(void)
Returns two character ISO-639 language descriptor for UI language.
Definition: mythcorecontext.cpp:1781
kMSRunShell
@ kMSRunShell
run process through shell
Definition: mythsystem.h:41
MetaGrabberScript::m_version
float m_version
Definition: metadatagrabber.h:90
MetaGrabberScript::FromInetref
static MetaGrabberScript FromInetref(const QString &inetref, bool absolute=false)
Definition: metadatagrabber.cpp:226
grabberTypes
static const QMap< GrabberType, GrabberOpts > grabberTypes
Definition: metadatagrabber.cpp:33
mythcorecontext.h
MetaGrabberScript::m_valid
bool m_valid
Definition: metadatagrabber.h:91
kGrabberGame
@ kGrabberGame
Definition: metadatagrabber.h:25
MetadataLookup::GetInetref
QString GetInetref() const
Definition: metadatacommon.h:355
MetaGrabberScript::LookupData
MetadataLookupList LookupData(const QString &inetref, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:490
grabberTypeStrings
static QMap< QString, GrabberType > grabberTypeStrings
Definition: metadatagrabber.cpp:48
MetaGrabberScript::GetType
GrabberType GetType(void) const
Definition: metadatagrabber.h:64
MetaGrabberScript::m_typestring
QString m_typestring
Definition: metadatagrabber.h:87
ParseMetadataItem
MetadataLookup * ParseMetadataItem(const QDomElement &item, MetadataLookup *lookup, bool passseas)
Definition: metadatacommon.cpp:895
MetaGrabberScript::MetaGrabberScript
MetaGrabberScript()=default
GrabberOpts
Definition: metadatagrabber.cpp:27
kGrabberAll
@ kGrabberAll
Definition: metadatagrabber.h:21
exitcodes.h
listLock
static QMutex listLock
Definition: mythsystemunix.cpp:62
kMSStdOut
@ kMSStdOut
allow access to stdout
Definition: mythsystem.h:39
metadatacommon.h
MythCoreContext::GetSetting
QString GetSetting(const QString &key, const QString &defaultval="")
Definition: mythcorecontext.cpp:916
MythSystemLegacy::Wait
uint Wait(time_t timeout=0)
Definition: mythsystemlegacy.cpp:242