MythTV  master
metadatagrabber.cpp
Go to the documentation of this file.
1 // Qt headers
2 #include <QDateTime>
3 #include <QDir>
4 #include <QMap>
5 #include <QMutex>
6 #include <QMutexLocker>
7 #include <QRegularExpression>
8 #include <utility>
9 
10 // MythTV headers
11 #include "libmythbase/exitcodes.h"
13 #include "libmythbase/mythdate.h"
14 #include "libmythbase/mythdirs.h"
17 
18 #include "metadatacommon.h"
19 #include "metadatagrabber.h"
20 
21 #define LOC QString("Metadata Grabber: ")
22 static constexpr std::chrono::seconds kGrabberRefresh { 60s };
23 
24 static const QRegularExpression kRetagRef { R"(^([a-zA-Z0-9_\-\.]+\.[a-zA-Z0-9]{1,3})[:_](.*))" };
25 
27 static QMutex s_grabberLock;
28 static QDateTime s_grabberAge;
29 
30 struct GrabberOpts {
31  QString m_path;
32  QString m_setting;
33  QString m_def;
34 };
35 
36 static const QMap<GrabberType, GrabberOpts> grabberTypes {
37  { kGrabberMovie, { "%1metadata/Movie/",
38  "MovieGrabber",
39  "metadata/Movie/tmdb3.py" } },
40  { kGrabberTelevision, { "%1metadata/Television/",
41  "TelevisionGrabber",
42  "metadata/Television/ttvdb4.py" } },
43  { kGrabberGame, { "%1metadata/Game/",
44  "mythgame.MetadataGrabber",
45  "metadata/Game/giantbomb.py" } },
46  { kGrabberMusic, { "%1metadata/Music",
47  "",
48  "" } }
49 };
50 
51 static QMap<QString, GrabberType> grabberTypeStrings {
52  { "movie", kGrabberMovie },
53  { "television", kGrabberTelevision },
54  { "game", kGrabberGame },
55  { "music", kGrabberMusic }
56 };
57 
59 {
60  return MetaGrabberScript::GetList(kGrabberAll, refresh);
61 }
62 
63 GrabberList MetaGrabberScript::GetList(const QString &type, bool refresh)
64 {
65  QString tmptype = type.toLower();
66  if (!grabberTypeStrings.contains(tmptype))
67  // unknown type, return empty list
68  return {};
69 
70  return MetaGrabberScript::GetList(grabberTypeStrings[tmptype], refresh);
71 }
72 
74  bool refresh)
75 {
76  GrabberList tmpGrabberList;
77  GrabberList retGrabberList;
78  {
79  QMutexLocker listLock(&s_grabberLock);
80  QDateTime now = MythDate::current();
81 
82  // refresh grabber scripts every 60 seconds
83  // this might have to be revised, or made more intelligent if
84  // the delay during refreshes is too great
85  if (refresh || !s_grabberAge.isValid() ||
86  (s_grabberAge.secsTo(now) > kGrabberRefresh.count()))
87  {
88  s_grabberList.clear();
89  LOG(VB_GENERAL, LOG_DEBUG, LOC + "Clearing grabber cache");
90 
91  // loop through different types of grabber scripts and the
92  // directories they are stored in
93  for (const auto& grabberType : std::as_const(grabberTypes))
94  {
95  QString path = (grabberType.m_path).arg(GetShareDir());
96  QDir dir = QDir(path);
97  if (!dir.exists())
98  {
99  LOG(VB_GENERAL, LOG_DEBUG, LOC +
100  QString("No script directory %1").arg(path));
101  continue;
102  }
103  QStringList scripts = dir.entryList(QDir::Executable | QDir::Files);
104  LOG(VB_GENERAL, LOG_DEBUG, LOC +
105  QString("Found %1 scripts in %2").arg(scripts.count()).arg(path));
106  if (scripts.count() == 0)
107  // no scripts found
108  continue;
109 
110  // loop through discovered scripts
111  for (const auto& name : std::as_const(scripts))
112  {
113  QString cmd = QDir(path).filePath(name);
114  MetaGrabberScript script(cmd);
115 
116  if (script.IsValid())
117  {
118  LOG(VB_GENERAL, LOG_DEBUG, LOC + "Adding " + script.m_command);
119  s_grabberList.append(script);
120  }
121  else
122  {
123  LOG(VB_GENERAL, LOG_DEBUG, LOC + "Failed " + name);
124  }
125  }
126  }
127 
128  s_grabberAge = now;
129  }
130 
131  tmpGrabberList = s_grabberList;
132  }
133 
134  for (const auto& item : std::as_const(tmpGrabberList))
135  {
136  if ((type == kGrabberAll) || (item.GetType() == type))
137  retGrabberList.append(item);
138  }
139 
140  return retGrabberList;
141 }
142 
144  const MetadataLookup *lookup)
145 {
146  if (lookup &&
147  !lookup->GetInetref().isEmpty() &&
148  lookup->GetInetref() != "00000000")
149  {
150  // inetref is defined, see if we have a pre-defined grabber
151  MetaGrabberScript grabber = FromInetref(lookup->GetInetref());
152 
153  if (grabber.IsValid())
154  {
155  return grabber;
156  }
157  // matching grabber was not found, just use the default
158  // fall through
159  }
160 
161  auto grabber = GetType(defaultType);
162  if (!grabber.m_valid)
163  {
164  QString name = grabberTypes[defaultType].m_setting;
165  if (name.isEmpty())
166  name = QString("Type %1").arg(defaultType);
167  LOG(VB_GENERAL, LOG_INFO,
168  QString("Grabber '%1' is not configured. Do you need to set PYTHONPATH?").arg(name));
169  }
170  return grabber;
171 }
172 
174 {
175  QString tmptype = type.toLower();
176  if (!grabberTypeStrings.contains(tmptype))
177  // unknown type, return empty grabber
178  return {};
179 
181 }
182 
184 {
185  QString cmd = gCoreContext->GetSetting(grabberTypes[type].m_setting,
186  grabberTypes[type].m_def);
187 
188  if (cmd.isEmpty())
189  {
190  // should the python bindings had not been installed at any stage
191  // the settings could have been set to an empty string, so use default
192  cmd = grabberTypes[type].m_def;
193  }
194 
195  // just pull it from the cache
196  GrabberList list = GetList(type);
197  for (const auto& item : std::as_const(list))
198  if (item.GetPath().endsWith(cmd))
199  return item;
200 
201  // polling the cache will cause a refresh, so lets just grab and
202  // process the script directly
203  QString fullcmd = QString("%1%2").arg(GetShareDir(), cmd);
204  MetaGrabberScript script(fullcmd);
205 
206  if (script.IsValid())
207  {
208  return script;
209  }
210 
211  return {};
212 }
213 
215  bool absolute)
216 {
217  GrabberList list = GetList();
218 
219  // search for direct match on tag
220  for (const auto& item : std::as_const(list))
221  {
222  if (item.GetCommand() == tag)
223  {
224  return item;
225  }
226  }
227 
228  // no direct match. do we require a direct match? search for one that works
229  if (!absolute)
230  {
231  for (const auto& item : std::as_const(list))
232  {
233  if (item.Accepts(tag))
234  {
235  return item;
236  }
237  }
238  }
239 
240  // no working match. return a blank
241  return {};
242 }
243 
245  bool absolute)
246 {
247  static QMutex s_reLock;
248  QMutexLocker lock(&s_reLock);
249  QString tag;
250  auto match = kRetagRef.match(inetref);
251  if (match.hasMatch())
252  tag = match.captured(1);
253  if (!tag.isEmpty())
254  {
255  // match found, pull out the grabber
256  MetaGrabberScript script = MetaGrabberScript::FromTag(tag, absolute);
257  if (script.IsValid())
258  return script;
259  }
260 
261  // no working match, return a blank
262  return {};
263 }
264 
265 QString MetaGrabberScript::CleanedInetref(const QString &inetref)
266 {
267  static QMutex s_reLock;
268  QMutexLocker lock(&s_reLock);
269 
270  // try to strip grabber tag from inetref
271  auto match = kRetagRef.match(inetref);
272  if (match.hasMatch())
273  return match.captured(2);
274  return inetref;
275 }
276 
277 MetaGrabberScript::MetaGrabberScript(QString path, const QDomElement &dom) :
278  m_fullcommand(std::move(path))
279 {
280  ParseGrabberVersion(dom);
281 }
282 
284 {
285  ParseGrabberVersion(dom);
286 }
287 
289 {
290  if (path.isEmpty())
291  return;
292  m_fullcommand = path;
293  if (path[0] != '/')
294  m_fullcommand.prepend(QString("%1metadata").arg(GetShareDir()));
295 
296  MythSystemLegacy grabber(path, QStringList() << "-v",
298  grabber.Run();
299  if (grabber.Wait() != GENERIC_EXIT_OK)
300  // script failed
301  return;
302 
303  QByteArray result = grabber.ReadAll();
304  if (result.isEmpty())
305  // no output
306  return;
307 
308  QDomDocument doc;
309  doc.setContent(result, true);
310  QDomElement root = doc.documentElement();
311  if (root.isNull())
312  // no valid XML
313  return;
314 
315  ParseGrabberVersion(root);
316  if (m_name.isEmpty())
317  // XML not processed correctly
318  return;
319 
320  m_valid = true;
321 }
322 
324 {
325  if (this != &other)
326  {
327  m_name = other.m_name;
328  m_author = other.m_author;
329  m_thumbnail = other.m_thumbnail;
330  m_command = other.m_command;
332  m_type = other.m_type;
333  m_typestring = other.m_typestring;
335  m_accepts = other.m_accepts;
336  m_version = other.m_version;
337  m_valid = other.m_valid;
338  }
339 
340  return *this;
341 }
342 
343 
344 void MetaGrabberScript::ParseGrabberVersion(const QDomElement &item)
345 {
346  m_name = item.firstChildElement("name").text();
347  m_author = item.firstChildElement("author").text();
348  m_thumbnail = item.firstChildElement("thumbnail").text();
349  m_command = item.firstChildElement("command").text();
350  m_description = item.firstChildElement("description").text();
351  m_version = item.firstChildElement("version").text().toFloat();
352  m_typestring = item.firstChildElement("type").text().toLower();
353 
354  if (!m_typestring.isEmpty() && grabberTypeStrings.contains(m_typestring))
356  else
358 
359  QDomElement accepts = item.firstChildElement("accepts");
360  if (!accepts.isNull())
361  {
362  while (!accepts.isNull())
363  {
364  m_accepts.append(accepts.text());
365  accepts = accepts.nextSiblingElement("accepts");
366  }
367  }
368 }
369 
371 {
372  if (!m_valid || m_fullcommand.isEmpty())
373  return false;
374 
375  QStringList args; args << "-t";
377 
378  grabber.Run();
379  return grabber.Wait() == GENERIC_EXIT_OK;
380 }
381 
382 // TODO
383 // using the MetadataLookup object as both argument input, and parsed output,
384 // is clumsy. break the inputs out into a separate object, and spawn a new
385 // MetadataLookup object in ParseMetadataItem, rather than requiring an
386 // existing one to reuse.
388  MetadataLookup *lookup, bool passseas)
389 {
391  MetadataLookupList list;
392 
393  LOG(VB_GENERAL, LOG_INFO, QString("Running Grabber: %1 %2")
394  .arg(m_fullcommand, args.join(" ")));
395 
396  grabber.Run();
397  if (grabber.Wait(180s) != GENERIC_EXIT_OK)
398  return list;
399 
400  QByteArray result = grabber.ReadAll();
401  if (!result.isEmpty())
402  {
403  QDomDocument doc;
404  doc.setContent(result, true);
405  QDomElement root = doc.documentElement();
406  QDomElement item = root.firstChildElement("item");
407 
408  while (!item.isNull())
409  {
410  MetadataLookup *tmp = ParseMetadataItem(item, lookup, passseas);
411  tmp->SetInetref(QString("%1_%2").arg(m_command,tmp->GetInetref()));
412  if (!tmp->GetCollectionref().isEmpty())
413  {
414  tmp->SetCollectionref(QString("%1_%2")
415  .arg(m_command, tmp->GetCollectionref()));
416  }
417  list.append(tmp);
418  // MetadataLookup is to be owned by the list
419  tmp->DecrRef();
420  item = item.nextSiblingElement("item");
421  }
422  }
423  return list;
424 }
425 
426 QString MetaGrabberScript::GetRelPath(void) const
427 {
428  QString share = GetShareDir();
429  if (m_fullcommand.startsWith(share))
430  return m_fullcommand.right(m_fullcommand.size() - share.size());
431 
432  return {};
433 }
434 
435 void MetaGrabberScript::toMap(InfoMap &metadataMap) const
436 {
437  metadataMap["name"] = m_name;
438  metadataMap["author"] = m_author;
439  metadataMap["thumbnailfilename"] = m_thumbnail;
440  metadataMap["command"] = m_command;
441  metadataMap["description"] = m_description;
442  metadataMap["version"] = QString::number(m_version);
443  metadataMap["type"] = m_typestring;
444 }
445 
447 {
448  args << "-l"
450  << "-a"
452 }
453 
455  MetadataLookup *lookup, bool passseas)
456 {
457  QStringList args;
459 
460  args << "-M"
461  << title;
462 
463  return RunGrabber(args, lookup, passseas);
464 }
465 
467  const QString &subtitle, MetadataLookup *lookup,
468  bool passseas)
469 {
470  QStringList args;
472 
473  args << "-N"
474  << title
475  << subtitle;
476 
477  return RunGrabber(args, lookup, passseas);
478 }
479 
481  [[maybe_unused]] const QString &title,
482  const QString &subtitle,
483  MetadataLookup *lookup, bool passseas)
484 {
485  QStringList args;
487 
488  args << "-N"
489  << CleanedInetref(inetref)
490  << subtitle;
491 
492  return RunGrabber(args, lookup, passseas);
493 }
494 
496  MetadataLookup *lookup, bool passseas)
497 {
498  QStringList args;
500 
501  args << "-D"
502  << CleanedInetref(inetref);
503 
504  return RunGrabber(args, lookup, passseas);
505 }
506 
508  int season, int episode, MetadataLookup *lookup,
509  bool passseas)
510 {
511  QStringList args;
513 
514  args << "-D"
515  << CleanedInetref(inetref)
516  << QString::number(season)
517  << QString::number(episode);
518 
519  return RunGrabber(args, lookup, passseas);
520 }
521 
523  const QString &collectionref, MetadataLookup *lookup,
524  bool passseas)
525 {
526  QStringList args;
528 
529  args << "-C"
530  << CleanedInetref(collectionref);
531 
532  return RunGrabber(args, lookup, passseas);
533 }
LOC
#define LOC
Definition: metadatagrabber.cpp:21
build_compdb.args
args
Definition: build_compdb.py:11
s_grabberLock
static QMutex s_grabberLock
Definition: metadatagrabber.cpp:27
MetaGrabberScript
Definition: metadatagrabber.h:29
MetaGrabberScript::m_fullcommand
QString m_fullcommand
Definition: metadatagrabber.h:84
GrabberType
GrabberType
Definition: metadatagrabber.h:20
MetaGrabberScript::LookupCollection
MetadataLookupList LookupCollection(const QString &collectionref, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:522
kGrabberTelevision
@ kGrabberTelevision
Definition: metadatagrabber.h:23
GrabberList
QList< MetaGrabberScript > GrabberList
Definition: metadatagrabber.h:18
MetaGrabberScript::GetList
static GrabberList GetList(bool refresh=false)
Definition: metadatagrabber.cpp:58
MythSystemLegacy
Definition: mythsystemlegacy.h:67
GrabberOpts::m_setting
QString m_setting
Definition: metadatagrabber.cpp:32
MetaGrabberScript::Test
bool Test(void)
Definition: metadatagrabber.cpp:370
MetaGrabberScript::SearchSubtitle
MetadataLookupList SearchSubtitle(const QString &title, const QString &subtitle, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:466
MythCoreContext::GetLocale
MythLocale * GetLocale(void) const
Definition: mythcorecontext.cpp:1758
MetaGrabberScript::ParseGrabberVersion
void ParseGrabberVersion(const QDomElement &item)
Definition: metadatagrabber.cpp:344
s_grabberAge
static QDateTime s_grabberAge
Definition: metadatagrabber.cpp:28
MetaGrabberScript::m_accepts
QStringList m_accepts
Definition: metadatagrabber.h:89
MetaGrabberScript::GetRelPath
QString GetRelPath(void) const
Definition: metadatagrabber.cpp:426
MetaGrabberScript::IsValid
bool IsValid(void) const
Definition: metadatagrabber.h:55
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:39
MythSystemLegacy::ReadAll
QByteArray & ReadAll()
Definition: mythsystemlegacy.cpp:402
MetaGrabberScript::toMap
void toMap(InfoMap &metadataMap) const
Definition: metadatagrabber.cpp:435
MetaGrabberScript::m_type
GrabberType m_type
Definition: metadatagrabber.h:86
mythdirs.h
MetaGrabberScript::m_thumbnail
QString m_thumbnail
Definition: metadatagrabber.h:83
GENERIC_EXIT_OK
@ GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:11
MetaGrabberScript::SetDefaultArgs
static void SetDefaultArgs(QStringList &args)
Definition: metadatagrabber.cpp:446
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:14
metadatagrabber.h
tmp
static guint32 * tmp
Definition: goom_core.cpp:26
kGrabberMovie
@ kGrabberMovie
Definition: metadatagrabber.h:22
GrabberOpts::m_path
QString m_path
Definition: metadatagrabber.cpp:31
mythsystemlegacy.h
MetadataLookup
Definition: metadatacommon.h:87
InfoMap
QHash< QString, QString > InfoMap
Definition: mythtypes.h:15
kGrabberMusic
@ kGrabberMusic
Definition: metadatagrabber.h:24
MetaGrabberScript::CleanedInetref
static QString CleanedInetref(const QString &inetref)
Definition: metadatagrabber.cpp:265
mythdate.h
MetaGrabberScript::Search
MetadataLookupList Search(const QString &title, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:454
mythlogging.h
MythLocale::GetCountryCode
QString GetCountryCode() const
Definition: mythlocale.cpp:57
MetaGrabberScript::m_command
QString m_command
Definition: metadatagrabber.h:85
RefCountedList< MetadataLookup >
kGrabberRefresh
static constexpr std::chrono::seconds kGrabberRefresh
Definition: metadatagrabber.cpp:22
GrabberOpts::m_def
QString m_def
Definition: metadatagrabber.cpp:33
GetShareDir
QString GetShareDir(void)
Definition: mythdirs.cpp:254
MetaGrabberScript::m_name
QString m_name
Definition: metadatagrabber.h:81
MythSystemLegacy::Wait
uint Wait(std::chrono::seconds timeout=0s)
Definition: mythsystemlegacy.cpp:243
MetaGrabberScript::FromTag
static MetaGrabberScript FromTag(const QString &tag, bool absolute=false)
Definition: metadatagrabber.cpp:214
MetaGrabberScript::operator=
MetaGrabberScript & operator=(const MetaGrabberScript &other)
Definition: metadatagrabber.cpp:323
MetaGrabberScript::m_author
QString m_author
Definition: metadatagrabber.h:82
MetaGrabberScript::m_description
QString m_description
Definition: metadatagrabber.h:88
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:55
MetaGrabberScript::RunGrabber
MetadataLookupList RunGrabber(const QStringList &args, MetadataLookup *lookup, bool passseas)
Definition: metadatagrabber.cpp:387
MetaGrabberScript::GetGrabber
static MetaGrabberScript GetGrabber(GrabberType defaultType, const MetadataLookup *lookup=nullptr)
Definition: metadatagrabber.cpp:143
kRetagRef
static const QRegularExpression kRetagRef
Definition: metadatagrabber.cpp:24
MythCoreContext::GetLanguage
QString GetLanguage(void)
Returns two character ISO-639 language descriptor for UI language.
Definition: mythcorecontext.cpp:1767
kMSRunShell
@ kMSRunShell
run process through shell
Definition: mythsystem.h:43
MetaGrabberScript::m_version
float m_version
Definition: metadatagrabber.h:90
MetaGrabberScript::FromInetref
static MetaGrabberScript FromInetref(const QString &inetref, bool absolute=false)
Definition: metadatagrabber.cpp:244
grabberTypes
static const QMap< GrabberType, GrabberOpts > grabberTypes
Definition: metadatagrabber.cpp:36
mythcorecontext.h
MetaGrabberScript::m_valid
bool m_valid
Definition: metadatagrabber.h:91
kGrabberInvalid
@ kGrabberInvalid
Definition: metadatagrabber.h:26
kGrabberGame
@ kGrabberGame
Definition: metadatagrabber.h:25
MetadataLookup::GetInetref
QString GetInetref() const
Definition: metadatacommon.h:356
MetaGrabberScript::LookupData
MetadataLookupList LookupData(const QString &inetref, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:495
std
Definition: mythchrono.h:23
grabberTypeStrings
static QMap< QString, GrabberType > grabberTypeStrings
Definition: metadatagrabber.cpp:51
MetaGrabberScript::GetType
GrabberType GetType(void) const
Definition: metadatagrabber.h:64
MetaGrabberScript::m_typestring
QString m_typestring
Definition: metadatagrabber.h:87
ParseMetadataItem
MetadataLookup * ParseMetadataItem(const QDomElement &item, MetadataLookup *lookup, bool passseas)
Definition: metadatacommon.cpp:894
s_grabberList
static GrabberList s_grabberList
Definition: metadatagrabber.cpp:26
MetaGrabberScript::MetaGrabberScript
MetaGrabberScript()=default
MythSystemLegacy::Run
void Run(std::chrono::seconds timeout=0s)
Runs a command inside the /bin/sh shell. Returns immediately.
Definition: mythsystemlegacy.cpp:213
GrabberOpts
Definition: metadatagrabber.cpp:30
kGrabberAll
@ kGrabberAll
Definition: metadatagrabber.h:21
exitcodes.h
listLock
static QMutex listLock
Definition: mythsystemunix.cpp:59
kMSStdOut
@ kMSStdOut
allow access to stdout
Definition: mythsystem.h:41
metadatacommon.h
MythCoreContext::GetSetting
QString GetSetting(const QString &key, const QString &defaultval="")
Definition: mythcorecontext.cpp:898