MythTV  master
metadatagrabber.cpp
Go to the documentation of this file.
1 // Qt headers
2 #include <QDateTime>
3 #include <QDir>
4 #include <QMap>
5 #include <QMutex>
6 #include <QMutexLocker>
7 #include <QRegularExpression>
8 #include <utility>
9 
10 // MythTV headers
11 #include "metadatagrabber.h"
12 #include "metadatacommon.h"
13 #include "mythsystemlegacy.h"
14 #include "exitcodes.h"
15 #include "mythdate.h"
16 #include "mythdirs.h"
17 #include "mythlogging.h"
18 #include "mythcorecontext.h"
19 
20 #define LOC QString("Metadata Grabber: ")
21 static constexpr std::chrono::seconds kGrabberRefresh { 60s };
22 
23 static const QRegularExpression kRetagRef { R"(^([a-zA-Z0-9_\-\.]+\.[a-zA-Z0-9]{1,3})[:_](.*))" };
24 
26 static QMutex s_grabberLock;
27 static QDateTime s_grabberAge;
28 
29 struct GrabberOpts {
30  QString m_path;
31  QString m_setting;
32  QString m_def;
33 };
34 
35 static const QMap<GrabberType, GrabberOpts> grabberTypes {
36  { kGrabberMovie, { "%1metadata/Movie/",
37  "MovieGrabber",
38  "metadata/Movie/tmdb3.py" } },
39  { kGrabberTelevision, { "%1metadata/Television/",
40  "TelevisionGrabber",
41  "metadata/Television/ttvdb.py" } },
42  { kGrabberGame, { "%1metadata/Game/",
43  "mythgame.MetadataGrabber",
44  "metadata/Game/giantbomb.py" } },
45  { kGrabberMusic, { "%1metadata/Music",
46  "",
47  "" } }
48 };
49 
50 static QMap<QString, GrabberType> grabberTypeStrings {
51  { "movie", kGrabberMovie },
52  { "television", kGrabberTelevision },
53  { "game", kGrabberGame },
54  { "music", kGrabberMusic }
55 };
56 
58 {
59  return MetaGrabberScript::GetList(kGrabberAll, refresh);
60 }
61 
62 GrabberList MetaGrabberScript::GetList(const QString &type, bool refresh)
63 {
64  QString tmptype = type.toLower();
65  if (!grabberTypeStrings.contains(tmptype))
66  // unknown type, return empty list
67  return GrabberList();
68 
69  return MetaGrabberScript::GetList(grabberTypeStrings[tmptype], refresh);
70 }
71 
73  bool refresh)
74 {
75  GrabberList tmpGrabberList;
76  GrabberList retGrabberList;
77  {
78  QMutexLocker listLock(&s_grabberLock);
79  QDateTime now = MythDate::current();
80 
81  // refresh grabber scripts every 60 seconds
82  // this might have to be revised, or made more intelligent if
83  // the delay during refreshes is too great
84  if (refresh || !s_grabberAge.isValid() ||
85  (s_grabberAge.secsTo(now) > kGrabberRefresh.count()))
86  {
87  s_grabberList.clear();
88  LOG(VB_GENERAL, LOG_DEBUG, LOC + "Clearing grabber cache");
89 
90  // loop through different types of grabber scripts and the
91  // directories they are stored in
92  for (const auto& grabberType : qAsConst(grabberTypes))
93  {
94  QString path = (grabberType.m_path).arg(GetShareDir());
95  QStringList scripts = QDir(path).entryList(QDir::Executable | QDir::Files);
96  if (scripts.count() == 0)
97  // no scripts found
98  continue;
99 
100  // loop through discovered scripts
101  for (const auto& name : qAsConst(scripts))
102  {
103  QString cmd = QDir(path).filePath(name);
104  MetaGrabberScript script(cmd);
105 
106  if (script.IsValid())
107  {
108  LOG(VB_GENERAL, LOG_DEBUG, LOC + "Adding " + script.m_command);
109  s_grabberList.append(script);
110  }
111  }
112  }
113 
114  s_grabberAge = now;
115  }
116 
117  tmpGrabberList = s_grabberList;
118  }
119 
120  for (const auto& item : qAsConst(tmpGrabberList))
121  {
122  if ((type == kGrabberAll) || (item.GetType() == type))
123  retGrabberList.append(item);
124  }
125 
126  return retGrabberList;
127 }
128 
130  const MetadataLookup *lookup)
131 {
132  if (!lookup)
133  {
134  return GetType(defaultType);
135  }
136 
137  if (!lookup->GetInetref().isEmpty() &&
138  lookup->GetInetref() != "00000000")
139  {
140  // inetref is defined, see if we have a pre-defined grabber
141  MetaGrabberScript grabber = FromInetref(lookup->GetInetref());
142 
143  if (grabber.IsValid())
144  {
145  return grabber;
146  }
147  // matching grabber was not found, just use the default
148  // fall through
149  }
150 
151  return GetType(defaultType);
152 }
153 
155 {
156  QString tmptype = type.toLower();
157  if (!grabberTypeStrings.contains(tmptype))
158  // unknown type, return empty grabber
159  return MetaGrabberScript();
160 
162 }
163 
165 {
166  QString cmd = gCoreContext->GetSetting(grabberTypes[type].m_setting,
167  grabberTypes[type].m_def);
168 
169  if (cmd.isEmpty())
170  {
171  // should the python bindings had not been installed at any stage
172  // the settings could have been set to an empty string, so use default
173  cmd = grabberTypes[type].m_def;
174  }
175 
177  {
178  // just pull it from the cache
179  GrabberList list = GetList();
180  for (const auto& item : qAsConst(list))
181  if (item.GetPath().endsWith(cmd))
182  return item;
183  }
184 
185  // polling the cache will cause a refresh, so lets just grab and
186  // process the script directly
187  QString fullcmd = QString("%1%2").arg(GetShareDir(), cmd);
188  MetaGrabberScript script(fullcmd);
189 
190  if (script.IsValid())
191  {
192  return script;
193  }
194 
195  return MetaGrabberScript();
196 }
197 
199  bool absolute)
200 {
201  GrabberList list = GetList();
202 
203  // search for direct match on tag
204  for (const auto& item : qAsConst(list))
205  {
206  if (item.GetCommand() == tag)
207  {
208  return item;
209  }
210  }
211 
212  // no direct match. do we require a direct match? search for one that works
213  if (!absolute)
214  {
215  for (const auto& item : qAsConst(list))
216  {
217  if (item.Accepts(tag))
218  {
219  return item;
220  }
221  }
222  }
223 
224  // no working match. return a blank
225  return MetaGrabberScript();
226 }
227 
229  bool absolute)
230 {
231  static QMutex s_reLock;
232  QMutexLocker lock(&s_reLock);
233  QString tag;
234  auto match = kRetagRef.match(inetref);
235  if (match.hasMatch())
236  tag = match.captured(1);
237  if (!tag.isEmpty())
238  {
239  // match found, pull out the grabber
240  MetaGrabberScript script = MetaGrabberScript::FromTag(tag, absolute);
241  if (script.IsValid())
242  return script;
243  }
244 
245  // no working match, return a blank
246  return MetaGrabberScript();
247 }
248 
249 QString MetaGrabberScript::CleanedInetref(const QString &inetref)
250 {
251  static QMutex s_reLock;
252  QMutexLocker lock(&s_reLock);
253 
254  // try to strip grabber tag from inetref
255  auto match = kRetagRef.match(inetref);
256  if (match.hasMatch())
257  return match.captured(2);
258  return inetref;
259 }
260 
261 MetaGrabberScript::MetaGrabberScript(QString path, const QDomElement &dom) :
262  m_fullcommand(std::move(path))
263 {
264  ParseGrabberVersion(dom);
265 }
266 
268 {
269  ParseGrabberVersion(dom);
270 }
271 
273 {
274  if (path.isEmpty())
275  return;
276  m_fullcommand = path;
277  if (path[0] != '/')
278  m_fullcommand.prepend(QString("%1metadata").arg(GetShareDir()));
279 
280  MythSystemLegacy grabber(path, QStringList() << "-v",
282  grabber.Run();
283  if (grabber.Wait() != GENERIC_EXIT_OK)
284  // script failed
285  return;
286 
287  QByteArray result = grabber.ReadAll();
288  if (result.isEmpty())
289  // no output
290  return;
291 
292  QDomDocument doc;
293  doc.setContent(result, true);
294  QDomElement root = doc.documentElement();
295  if (root.isNull())
296  // no valid XML
297  return;
298 
299  ParseGrabberVersion(root);
300  if (m_name.isEmpty())
301  // XML not processed correctly
302  return;
303 
304  m_valid = true;
305 }
306 
308 {
309  if (this != &other)
310  {
311  m_name = other.m_name;
312  m_author = other.m_author;
313  m_thumbnail = other.m_thumbnail;
314  m_command = other.m_command;
316  m_type = other.m_type;
317  m_typestring = other.m_typestring;
319  m_accepts = other.m_accepts;
320  m_version = other.m_version;
321  m_valid = other.m_valid;
322  }
323 
324  return *this;
325 }
326 
327 
328 void MetaGrabberScript::ParseGrabberVersion(const QDomElement &item)
329 {
330  m_name = item.firstChildElement("name").text();
331  m_author = item.firstChildElement("author").text();
332  m_thumbnail = item.firstChildElement("thumbnail").text();
333  m_command = item.firstChildElement("command").text();
334  m_description = item.firstChildElement("description").text();
335  m_version = item.firstChildElement("version").text().toFloat();
336  m_typestring = item.firstChildElement("type").text().toLower();
337 
338  if (!m_typestring.isEmpty() && grabberTypeStrings.contains(m_typestring))
340  else
342 
343  QDomElement accepts = item.firstChildElement("accepts");
344  if (!accepts.isNull())
345  {
346  while (!accepts.isNull())
347  {
348  m_accepts.append(accepts.text());
349  accepts = accepts.nextSiblingElement("accepts");
350  }
351  }
352 }
353 
355 {
356  if (!m_valid || m_fullcommand.isEmpty())
357  return false;
358 
359  QStringList args; args << "-t";
361 
362  grabber.Run();
363  return grabber.Wait() == GENERIC_EXIT_OK;
364 }
365 
366 // TODO
367 // using the MetadataLookup object as both argument input, and parsed output,
368 // is clumsy. break the inputs out into a separate object, and spawn a new
369 // MetadataLookup object in ParseMetadataItem, rather than requiring an
370 // existing one to reuse.
372  MetadataLookup *lookup, bool passseas)
373 {
375  MetadataLookupList list;
376 
377  LOG(VB_GENERAL, LOG_INFO, QString("Running Grabber: %1 %2")
378  .arg(m_fullcommand, args.join(" ")));
379 
380  grabber.Run();
381  if (grabber.Wait(180s) != GENERIC_EXIT_OK)
382  return list;
383 
384  QByteArray result = grabber.ReadAll();
385  if (!result.isEmpty())
386  {
387  QDomDocument doc;
388  doc.setContent(result, true);
389  QDomElement root = doc.documentElement();
390  QDomElement item = root.firstChildElement("item");
391 
392  while (!item.isNull())
393  {
394  MetadataLookup *tmp = ParseMetadataItem(item, lookup, passseas);
395  tmp->SetInetref(QString("%1_%2").arg(m_command,tmp->GetInetref()));
396  if (!tmp->GetCollectionref().isEmpty())
397  {
398  tmp->SetCollectionref(QString("%1_%2")
399  .arg(m_command, tmp->GetCollectionref()));
400  }
401  list.append(tmp);
402  // MetadataLookup is to be owned by the list
403  tmp->DecrRef();
404  item = item.nextSiblingElement("item");
405  }
406  }
407  return list;
408 }
409 
410 QString MetaGrabberScript::GetRelPath(void) const
411 {
412  QString share = GetShareDir();
413  if (m_fullcommand.startsWith(share))
414  return m_fullcommand.right(m_fullcommand.size() - share.size());
415 
416  return QString();
417 }
418 
419 void MetaGrabberScript::toMap(InfoMap &metadataMap) const
420 {
421  metadataMap["name"] = m_name;
422  metadataMap["author"] = m_author;
423  metadataMap["thumbnailfilename"] = m_thumbnail;
424  metadataMap["command"] = m_command;
425  metadataMap["description"] = m_description;
426  metadataMap["version"] = QString::number(m_version);
427  metadataMap["type"] = m_typestring;
428 }
429 
431 {
432  args << "-l"
434  << "-a"
436 }
437 
439  MetadataLookup *lookup, bool passseas)
440 {
441  QStringList args;
443 
444  args << "-M"
445  << title;
446 
447  return RunGrabber(args, lookup, passseas);
448 }
449 
451  const QString &subtitle, MetadataLookup *lookup,
452  bool passseas)
453 {
454  QStringList args;
456 
457  args << "-N"
458  << title
459  << subtitle;
460 
461  return RunGrabber(args, lookup, passseas);
462 }
463 
465  const QString &title, const QString &subtitle,
466  MetadataLookup *lookup, bool passseas)
467 {
468  (void)title;
469  QStringList args;
471 
472  args << "-N"
473  << CleanedInetref(inetref)
474  << subtitle;
475 
476  return RunGrabber(args, lookup, passseas);
477 }
478 
480  MetadataLookup *lookup, bool passseas)
481 {
482  QStringList args;
484 
485  args << "-D"
486  << CleanedInetref(inetref);
487 
488  return RunGrabber(args, lookup, passseas);
489 }
490 
492  int season, int episode, MetadataLookup *lookup,
493  bool passseas)
494 {
495  QStringList args;
497 
498  args << "-D"
499  << CleanedInetref(inetref)
500  << QString::number(season)
501  << QString::number(episode);
502 
503  return RunGrabber(args, lookup, passseas);
504 }
505 
507  const QString &collectionref, MetadataLookup *lookup,
508  bool passseas)
509 {
510  QStringList args;
512 
513  args << "-C"
514  << CleanedInetref(collectionref);
515 
516  return RunGrabber(args, lookup, passseas);
517 }
LOC
#define LOC
Definition: metadatagrabber.cpp:20
build_compdb.args
args
Definition: build_compdb.py:11
s_grabberLock
static QMutex s_grabberLock
Definition: metadatagrabber.cpp:26
MetaGrabberScript
Definition: metadatagrabber.h:29
MetaGrabberScript::m_fullcommand
QString m_fullcommand
Definition: metadatagrabber.h:84
GrabberType
GrabberType
Definition: metadatagrabber.h:20
MetaGrabberScript::LookupCollection
MetadataLookupList LookupCollection(const QString &collectionref, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:506
GENERIC_EXIT_OK
#define GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:10
kGrabberTelevision
@ kGrabberTelevision
Definition: metadatagrabber.h:23
GrabberList
QList< MetaGrabberScript > GrabberList
Definition: metadatagrabber.h:18
MetaGrabberScript::GetList
static GrabberList GetList(bool refresh=false)
Definition: metadatagrabber.cpp:57
MythSystemLegacy
Definition: mythsystemlegacy.h:67
GrabberOpts::m_setting
QString m_setting
Definition: metadatagrabber.cpp:31
MetaGrabberScript::Test
bool Test(void)
Definition: metadatagrabber.cpp:354
MetaGrabberScript::SearchSubtitle
MetadataLookupList SearchSubtitle(const QString &title, const QString &subtitle, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:450
MythCoreContext::GetLocale
MythLocale * GetLocale(void) const
Definition: mythcorecontext.cpp:1776
MetaGrabberScript::ParseGrabberVersion
void ParseGrabberVersion(const QDomElement &item)
Definition: metadatagrabber.cpp:328
s_grabberAge
static QDateTime s_grabberAge
Definition: metadatagrabber.cpp:27
MetaGrabberScript::m_accepts
QStringList m_accepts
Definition: metadatagrabber.h:89
MetaGrabberScript::GetRelPath
QString GetRelPath(void) const
Definition: metadatagrabber.cpp:410
MetaGrabberScript::IsValid
bool IsValid(void) const
Definition: metadatagrabber.h:55
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:23
MythSystemLegacy::ReadAll
QByteArray & ReadAll()
Definition: mythsystemlegacy.cpp:398
MetaGrabberScript::toMap
void toMap(InfoMap &metadataMap) const
Definition: metadatagrabber.cpp:419
MetaGrabberScript::m_type
GrabberType m_type
Definition: metadatagrabber.h:86
mythdirs.h
MetaGrabberScript::m_thumbnail
QString m_thumbnail
Definition: metadatagrabber.h:83
MetaGrabberScript::SetDefaultArgs
static void SetDefaultArgs(QStringList &args)
Definition: metadatagrabber.cpp:430
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:10
metadatagrabber.h
tmp
static guint32 * tmp
Definition: goom_core.cpp:31
kGrabberMovie
@ kGrabberMovie
Definition: metadatagrabber.h:22
GrabberOpts::m_path
QString m_path
Definition: metadatagrabber.cpp:30
mythsystemlegacy.h
MetadataLookup
Definition: metadatacommon.h:87
InfoMap
QHash< QString, QString > InfoMap
Definition: mythtypes.h:15
kGrabberMusic
@ kGrabberMusic
Definition: metadatagrabber.h:24
MetaGrabberScript::CleanedInetref
static QString CleanedInetref(const QString &inetref)
Definition: metadatagrabber.cpp:249
mythdate.h
MetaGrabberScript::Search
MetadataLookupList Search(const QString &title, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:438
mythlogging.h
MythLocale::GetCountryCode
QString GetCountryCode() const
Definition: mythlocale.cpp:57
MetaGrabberScript::m_command
QString m_command
Definition: metadatagrabber.h:85
RefCountedList< MetadataLookup >
kGrabberRefresh
static constexpr std::chrono::seconds kGrabberRefresh
Definition: metadatagrabber.cpp:21
GrabberOpts::m_def
QString m_def
Definition: metadatagrabber.cpp:32
GetShareDir
QString GetShareDir(void)
Definition: mythdirs.cpp:222
MetaGrabberScript::m_name
QString m_name
Definition: metadatagrabber.h:81
MythSystemLegacy::Wait
uint Wait(std::chrono::seconds timeout=0s)
Definition: mythsystemlegacy.cpp:242
MetaGrabberScript::FromTag
static MetaGrabberScript FromTag(const QString &tag, bool absolute=false)
Definition: metadatagrabber.cpp:198
MetaGrabberScript::operator=
MetaGrabberScript & operator=(const MetaGrabberScript &other)
Definition: metadatagrabber.cpp:307
MetaGrabberScript::m_author
QString m_author
Definition: metadatagrabber.h:82
MetaGrabberScript::m_description
QString m_description
Definition: metadatagrabber.h:88
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:60
MetaGrabberScript::RunGrabber
MetadataLookupList RunGrabber(const QStringList &args, MetadataLookup *lookup, bool passseas)
Definition: metadatagrabber.cpp:371
MetaGrabberScript::GetGrabber
static MetaGrabberScript GetGrabber(GrabberType defaultType, const MetadataLookup *lookup=nullptr)
Definition: metadatagrabber.cpp:129
kRetagRef
static const QRegularExpression kRetagRef
Definition: metadatagrabber.cpp:23
MythCoreContext::GetLanguage
QString GetLanguage(void)
Returns two character ISO-639 language descriptor for UI language.
Definition: mythcorecontext.cpp:1785
kMSRunShell
@ kMSRunShell
run process through shell
Definition: mythsystem.h:43
MetaGrabberScript::m_version
float m_version
Definition: metadatagrabber.h:90
MetaGrabberScript::FromInetref
static MetaGrabberScript FromInetref(const QString &inetref, bool absolute=false)
Definition: metadatagrabber.cpp:228
grabberTypes
static const QMap< GrabberType, GrabberOpts > grabberTypes
Definition: metadatagrabber.cpp:35
MythDate::secsInPast
std::chrono::seconds secsInPast(const QDateTime &past)
Definition: mythdate.cpp:199
mythcorecontext.h
MetaGrabberScript::m_valid
bool m_valid
Definition: metadatagrabber.h:91
kGrabberInvalid
@ kGrabberInvalid
Definition: metadatagrabber.h:26
kGrabberGame
@ kGrabberGame
Definition: metadatagrabber.h:25
MetadataLookup::GetInetref
QString GetInetref() const
Definition: metadatacommon.h:356
MetaGrabberScript::LookupData
MetadataLookupList LookupData(const QString &inetref, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:479
std
Definition: mythchrono.h:23
grabberTypeStrings
static QMap< QString, GrabberType > grabberTypeStrings
Definition: metadatagrabber.cpp:50
MetaGrabberScript::GetType
GrabberType GetType(void) const
Definition: metadatagrabber.h:64
MetaGrabberScript::m_typestring
QString m_typestring
Definition: metadatagrabber.h:87
ParseMetadataItem
MetadataLookup * ParseMetadataItem(const QDomElement &item, MetadataLookup *lookup, bool passseas)
Definition: metadatacommon.cpp:892
s_grabberList
static GrabberList s_grabberList
Definition: metadatagrabber.cpp:25
MetaGrabberScript::MetaGrabberScript
MetaGrabberScript()=default
MythSystemLegacy::Run
void Run(std::chrono::seconds timeout=0s)
Runs a command inside the /bin/sh shell. Returns immediately.
Definition: mythsystemlegacy.cpp:212
GrabberOpts
Definition: metadatagrabber.cpp:29
kGrabberAll
@ kGrabberAll
Definition: metadatagrabber.h:21
exitcodes.h
listLock
static QMutex listLock
Definition: mythsystemunix.cpp:61
kMSStdOut
@ kMSStdOut
allow access to stdout
Definition: mythsystem.h:41
metadatacommon.h
MythCoreContext::GetSetting
QString GetSetting(const QString &key, const QString &defaultval="")
Definition: mythcorecontext.cpp:922