MythTV  master
metadatagrabber.cpp
Go to the documentation of this file.
1 // Qt headers
2 #include <QDateTime>
3 #include <QDir>
4 #include <QMap>
5 #include <QMutex>
6 #include <QMutexLocker>
7 #include <QRegularExpression>
8 #include <utility>
9 
10 // MythTV headers
11 #include "libmythbase/exitcodes.h"
13 #include "libmythbase/mythdate.h"
14 #include "libmythbase/mythdirs.h"
17 
18 #include "metadatacommon.h"
19 #include "metadatagrabber.h"
20 
21 #define LOC QString("Metadata Grabber: ")
22 static constexpr std::chrono::seconds kGrabberRefresh { 60s };
23 
24 static const QRegularExpression kRetagRef { R"(^([a-zA-Z0-9_\-\.]+\.[a-zA-Z0-9]{1,3})[:_](.*))" };
25 
27 static QMutex s_grabberLock;
28 static QDateTime s_grabberAge;
29 
30 struct GrabberOpts {
31  QString m_path;
32  QString m_setting;
33  QString m_def;
34 };
35 
36 static const QMap<GrabberType, GrabberOpts> grabberTypes {
37  { kGrabberMovie, { "%1metadata/Movie/",
38  "MovieGrabber",
39  "metadata/Movie/tmdb3.py" } },
40  { kGrabberTelevision, { "%1metadata/Television/",
41  "TelevisionGrabber",
42  "metadata/Television/ttvdb4.py" } },
43  { kGrabberGame, { "%1metadata/Game/",
44  "mythgame.MetadataGrabber",
45  "metadata/Game/giantbomb.py" } },
46  { kGrabberMusic, { "%1metadata/Music",
47  "",
48  "" } }
49 };
50 
51 static QMap<QString, GrabberType> grabberTypeStrings {
52  { "movie", kGrabberMovie },
53  { "television", kGrabberTelevision },
54  { "game", kGrabberGame },
55  { "music", kGrabberMusic }
56 };
57 
59 {
60  return MetaGrabberScript::GetList(kGrabberAll, refresh);
61 }
62 
63 GrabberList MetaGrabberScript::GetList(const QString &type, bool refresh)
64 {
65  QString tmptype = type.toLower();
66  if (!grabberTypeStrings.contains(tmptype))
67  // unknown type, return empty list
68  return {};
69 
70  return MetaGrabberScript::GetList(grabberTypeStrings[tmptype], refresh);
71 }
72 
74  bool refresh)
75 {
76  GrabberList tmpGrabberList;
77  GrabberList retGrabberList;
78  {
79  QMutexLocker listLock(&s_grabberLock);
80  QDateTime now = MythDate::current();
81 
82  // refresh grabber scripts every 60 seconds
83  // this might have to be revised, or made more intelligent if
84  // the delay during refreshes is too great
85  if (refresh || !s_grabberAge.isValid() ||
86  (s_grabberAge.secsTo(now) > kGrabberRefresh.count()))
87  {
88  s_grabberList.clear();
89  LOG(VB_GENERAL, LOG_DEBUG, LOC + "Clearing grabber cache");
90 
91  // loop through different types of grabber scripts and the
92  // directories they are stored in
93  for (const auto& grabberType : qAsConst(grabberTypes))
94  {
95  QString path = (grabberType.m_path).arg(GetShareDir());
96  QStringList scripts = QDir(path).entryList(QDir::Executable | QDir::Files);
97  if (scripts.count() == 0)
98  // no scripts found
99  continue;
100 
101  // loop through discovered scripts
102  for (const auto& name : qAsConst(scripts))
103  {
104  QString cmd = QDir(path).filePath(name);
105  MetaGrabberScript script(cmd);
106 
107  if (script.IsValid())
108  {
109  LOG(VB_GENERAL, LOG_DEBUG, LOC + "Adding " + script.m_command);
110  s_grabberList.append(script);
111  }
112  }
113  }
114 
115  s_grabberAge = now;
116  }
117 
118  tmpGrabberList = s_grabberList;
119  }
120 
121  for (const auto& item : qAsConst(tmpGrabberList))
122  {
123  if ((type == kGrabberAll) || (item.GetType() == type))
124  retGrabberList.append(item);
125  }
126 
127  return retGrabberList;
128 }
129 
131  const MetadataLookup *lookup)
132 {
133  if (!lookup)
134  {
135  return GetType(defaultType);
136  }
137 
138  if (!lookup->GetInetref().isEmpty() &&
139  lookup->GetInetref() != "00000000")
140  {
141  // inetref is defined, see if we have a pre-defined grabber
142  MetaGrabberScript grabber = FromInetref(lookup->GetInetref());
143 
144  if (grabber.IsValid())
145  {
146  return grabber;
147  }
148  // matching grabber was not found, just use the default
149  // fall through
150  }
151 
152  return GetType(defaultType);
153 }
154 
156 {
157  QString tmptype = type.toLower();
158  if (!grabberTypeStrings.contains(tmptype))
159  // unknown type, return empty grabber
160  return {};
161 
163 }
164 
166 {
167  QString cmd = gCoreContext->GetSetting(grabberTypes[type].m_setting,
168  grabberTypes[type].m_def);
169 
170  if (cmd.isEmpty())
171  {
172  // should the python bindings had not been installed at any stage
173  // the settings could have been set to an empty string, so use default
174  cmd = grabberTypes[type].m_def;
175  }
176 
178  {
179  // just pull it from the cache
180  GrabberList list = GetList();
181  for (const auto& item : qAsConst(list))
182  if (item.GetPath().endsWith(cmd))
183  return item;
184  }
185 
186  // polling the cache will cause a refresh, so lets just grab and
187  // process the script directly
188  QString fullcmd = QString("%1%2").arg(GetShareDir(), cmd);
189  MetaGrabberScript script(fullcmd);
190 
191  if (script.IsValid())
192  {
193  return script;
194  }
195 
196  return {};
197 }
198 
200  bool absolute)
201 {
202  GrabberList list = GetList();
203 
204  // search for direct match on tag
205  for (const auto& item : qAsConst(list))
206  {
207  if (item.GetCommand() == tag)
208  {
209  return item;
210  }
211  }
212 
213  // no direct match. do we require a direct match? search for one that works
214  if (!absolute)
215  {
216  for (const auto& item : qAsConst(list))
217  {
218  if (item.Accepts(tag))
219  {
220  return item;
221  }
222  }
223  }
224 
225  // no working match. return a blank
226  return {};
227 }
228 
230  bool absolute)
231 {
232  static QMutex s_reLock;
233  QMutexLocker lock(&s_reLock);
234  QString tag;
235  auto match = kRetagRef.match(inetref);
236  if (match.hasMatch())
237  tag = match.captured(1);
238  if (!tag.isEmpty())
239  {
240  // match found, pull out the grabber
241  MetaGrabberScript script = MetaGrabberScript::FromTag(tag, absolute);
242  if (script.IsValid())
243  return script;
244  }
245 
246  // no working match, return a blank
247  return {};
248 }
249 
250 QString MetaGrabberScript::CleanedInetref(const QString &inetref)
251 {
252  static QMutex s_reLock;
253  QMutexLocker lock(&s_reLock);
254 
255  // try to strip grabber tag from inetref
256  auto match = kRetagRef.match(inetref);
257  if (match.hasMatch())
258  return match.captured(2);
259  return inetref;
260 }
261 
262 MetaGrabberScript::MetaGrabberScript(QString path, const QDomElement &dom) :
263  m_fullcommand(std::move(path))
264 {
265  ParseGrabberVersion(dom);
266 }
267 
269 {
270  ParseGrabberVersion(dom);
271 }
272 
274 {
275  if (path.isEmpty())
276  return;
277  m_fullcommand = path;
278  if (path[0] != '/')
279  m_fullcommand.prepend(QString("%1metadata").arg(GetShareDir()));
280 
281  MythSystemLegacy grabber(path, QStringList() << "-v",
283  grabber.Run();
284  if (grabber.Wait() != GENERIC_EXIT_OK)
285  // script failed
286  return;
287 
288  QByteArray result = grabber.ReadAll();
289  if (result.isEmpty())
290  // no output
291  return;
292 
293  QDomDocument doc;
294  doc.setContent(result, true);
295  QDomElement root = doc.documentElement();
296  if (root.isNull())
297  // no valid XML
298  return;
299 
300  ParseGrabberVersion(root);
301  if (m_name.isEmpty())
302  // XML not processed correctly
303  return;
304 
305  m_valid = true;
306 }
307 
309 {
310  if (this != &other)
311  {
312  m_name = other.m_name;
313  m_author = other.m_author;
314  m_thumbnail = other.m_thumbnail;
315  m_command = other.m_command;
317  m_type = other.m_type;
318  m_typestring = other.m_typestring;
320  m_accepts = other.m_accepts;
321  m_version = other.m_version;
322  m_valid = other.m_valid;
323  }
324 
325  return *this;
326 }
327 
328 
329 void MetaGrabberScript::ParseGrabberVersion(const QDomElement &item)
330 {
331  m_name = item.firstChildElement("name").text();
332  m_author = item.firstChildElement("author").text();
333  m_thumbnail = item.firstChildElement("thumbnail").text();
334  m_command = item.firstChildElement("command").text();
335  m_description = item.firstChildElement("description").text();
336  m_version = item.firstChildElement("version").text().toFloat();
337  m_typestring = item.firstChildElement("type").text().toLower();
338 
339  if (!m_typestring.isEmpty() && grabberTypeStrings.contains(m_typestring))
341  else
343 
344  QDomElement accepts = item.firstChildElement("accepts");
345  if (!accepts.isNull())
346  {
347  while (!accepts.isNull())
348  {
349  m_accepts.append(accepts.text());
350  accepts = accepts.nextSiblingElement("accepts");
351  }
352  }
353 }
354 
356 {
357  if (!m_valid || m_fullcommand.isEmpty())
358  return false;
359 
360  QStringList args; args << "-t";
362 
363  grabber.Run();
364  return grabber.Wait() == GENERIC_EXIT_OK;
365 }
366 
367 // TODO
368 // using the MetadataLookup object as both argument input, and parsed output,
369 // is clumsy. break the inputs out into a separate object, and spawn a new
370 // MetadataLookup object in ParseMetadataItem, rather than requiring an
371 // existing one to reuse.
373  MetadataLookup *lookup, bool passseas)
374 {
376  MetadataLookupList list;
377 
378  LOG(VB_GENERAL, LOG_INFO, QString("Running Grabber: %1 %2")
379  .arg(m_fullcommand, args.join(" ")));
380 
381  grabber.Run();
382  if (grabber.Wait(180s) != GENERIC_EXIT_OK)
383  return list;
384 
385  QByteArray result = grabber.ReadAll();
386  if (!result.isEmpty())
387  {
388  QDomDocument doc;
389  doc.setContent(result, true);
390  QDomElement root = doc.documentElement();
391  QDomElement item = root.firstChildElement("item");
392 
393  while (!item.isNull())
394  {
395  MetadataLookup *tmp = ParseMetadataItem(item, lookup, passseas);
396  tmp->SetInetref(QString("%1_%2").arg(m_command,tmp->GetInetref()));
397  if (!tmp->GetCollectionref().isEmpty())
398  {
399  tmp->SetCollectionref(QString("%1_%2")
400  .arg(m_command, tmp->GetCollectionref()));
401  }
402  list.append(tmp);
403  // MetadataLookup is to be owned by the list
404  tmp->DecrRef();
405  item = item.nextSiblingElement("item");
406  }
407  }
408  return list;
409 }
410 
411 QString MetaGrabberScript::GetRelPath(void) const
412 {
413  QString share = GetShareDir();
414  if (m_fullcommand.startsWith(share))
415  return m_fullcommand.right(m_fullcommand.size() - share.size());
416 
417  return {};
418 }
419 
420 void MetaGrabberScript::toMap(InfoMap &metadataMap) const
421 {
422  metadataMap["name"] = m_name;
423  metadataMap["author"] = m_author;
424  metadataMap["thumbnailfilename"] = m_thumbnail;
425  metadataMap["command"] = m_command;
426  metadataMap["description"] = m_description;
427  metadataMap["version"] = QString::number(m_version);
428  metadataMap["type"] = m_typestring;
429 }
430 
432 {
433  args << "-l"
435  << "-a"
437 }
438 
440  MetadataLookup *lookup, bool passseas)
441 {
442  QStringList args;
444 
445  args << "-M"
446  << title;
447 
448  return RunGrabber(args, lookup, passseas);
449 }
450 
452  const QString &subtitle, MetadataLookup *lookup,
453  bool passseas)
454 {
455  QStringList args;
457 
458  args << "-N"
459  << title
460  << subtitle;
461 
462  return RunGrabber(args, lookup, passseas);
463 }
464 
466  [[maybe_unused]] const QString &title,
467  const QString &subtitle,
468  MetadataLookup *lookup, bool passseas)
469 {
470  QStringList args;
472 
473  args << "-N"
474  << CleanedInetref(inetref)
475  << subtitle;
476 
477  return RunGrabber(args, lookup, passseas);
478 }
479 
481  MetadataLookup *lookup, bool passseas)
482 {
483  QStringList args;
485 
486  args << "-D"
487  << CleanedInetref(inetref);
488 
489  return RunGrabber(args, lookup, passseas);
490 }
491 
493  int season, int episode, MetadataLookup *lookup,
494  bool passseas)
495 {
496  QStringList args;
498 
499  args << "-D"
500  << CleanedInetref(inetref)
501  << QString::number(season)
502  << QString::number(episode);
503 
504  return RunGrabber(args, lookup, passseas);
505 }
506 
508  const QString &collectionref, MetadataLookup *lookup,
509  bool passseas)
510 {
511  QStringList args;
513 
514  args << "-C"
515  << CleanedInetref(collectionref);
516 
517  return RunGrabber(args, lookup, passseas);
518 }
LOC
#define LOC
Definition: metadatagrabber.cpp:21
build_compdb.args
args
Definition: build_compdb.py:11
s_grabberLock
static QMutex s_grabberLock
Definition: metadatagrabber.cpp:27
MetaGrabberScript
Definition: metadatagrabber.h:29
MetaGrabberScript::m_fullcommand
QString m_fullcommand
Definition: metadatagrabber.h:84
GrabberType
GrabberType
Definition: metadatagrabber.h:20
MetaGrabberScript::LookupCollection
MetadataLookupList LookupCollection(const QString &collectionref, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:507
kGrabberTelevision
@ kGrabberTelevision
Definition: metadatagrabber.h:23
GrabberList
QList< MetaGrabberScript > GrabberList
Definition: metadatagrabber.h:18
MetaGrabberScript::GetList
static GrabberList GetList(bool refresh=false)
Definition: metadatagrabber.cpp:58
MythSystemLegacy
Definition: mythsystemlegacy.h:67
GrabberOpts::m_setting
QString m_setting
Definition: metadatagrabber.cpp:32
MetaGrabberScript::Test
bool Test(void)
Definition: metadatagrabber.cpp:355
MetaGrabberScript::SearchSubtitle
MetadataLookupList SearchSubtitle(const QString &title, const QString &subtitle, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:451
MythCoreContext::GetLocale
MythLocale * GetLocale(void) const
Definition: mythcorecontext.cpp:1761
MetaGrabberScript::ParseGrabberVersion
void ParseGrabberVersion(const QDomElement &item)
Definition: metadatagrabber.cpp:329
s_grabberAge
static QDateTime s_grabberAge
Definition: metadatagrabber.cpp:28
MetaGrabberScript::m_accepts
QStringList m_accepts
Definition: metadatagrabber.h:89
MetaGrabberScript::GetRelPath
QString GetRelPath(void) const
Definition: metadatagrabber.cpp:411
MetaGrabberScript::IsValid
bool IsValid(void) const
Definition: metadatagrabber.h:55
LOG
#define LOG(_MASK_, _LEVEL_, _QSTRING_)
Definition: mythlogging.h:39
MythSystemLegacy::ReadAll
QByteArray & ReadAll()
Definition: mythsystemlegacy.cpp:402
MetaGrabberScript::toMap
void toMap(InfoMap &metadataMap) const
Definition: metadatagrabber.cpp:420
MetaGrabberScript::m_type
GrabberType m_type
Definition: metadatagrabber.h:86
mythdirs.h
MetaGrabberScript::m_thumbnail
QString m_thumbnail
Definition: metadatagrabber.h:83
GENERIC_EXIT_OK
@ GENERIC_EXIT_OK
Exited with no error.
Definition: exitcodes.h:11
MetaGrabberScript::SetDefaultArgs
static void SetDefaultArgs(QStringList &args)
Definition: metadatagrabber.cpp:431
MythDate::current
QDateTime current(bool stripped)
Returns current Date and Time in UTC.
Definition: mythdate.cpp:14
metadatagrabber.h
tmp
static guint32 * tmp
Definition: goom_core.cpp:26
kGrabberMovie
@ kGrabberMovie
Definition: metadatagrabber.h:22
GrabberOpts::m_path
QString m_path
Definition: metadatagrabber.cpp:31
mythsystemlegacy.h
MetadataLookup
Definition: metadatacommon.h:87
InfoMap
QHash< QString, QString > InfoMap
Definition: mythtypes.h:15
kGrabberMusic
@ kGrabberMusic
Definition: metadatagrabber.h:24
MetaGrabberScript::CleanedInetref
static QString CleanedInetref(const QString &inetref)
Definition: metadatagrabber.cpp:250
mythdate.h
MetaGrabberScript::Search
MetadataLookupList Search(const QString &title, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:439
mythlogging.h
MythLocale::GetCountryCode
QString GetCountryCode() const
Definition: mythlocale.cpp:57
MetaGrabberScript::m_command
QString m_command
Definition: metadatagrabber.h:85
RefCountedList< MetadataLookup >
kGrabberRefresh
static constexpr std::chrono::seconds kGrabberRefresh
Definition: metadatagrabber.cpp:22
GrabberOpts::m_def
QString m_def
Definition: metadatagrabber.cpp:33
GetShareDir
QString GetShareDir(void)
Definition: mythdirs.cpp:254
MetaGrabberScript::m_name
QString m_name
Definition: metadatagrabber.h:81
MythSystemLegacy::Wait
uint Wait(std::chrono::seconds timeout=0s)
Definition: mythsystemlegacy.cpp:243
MetaGrabberScript::FromTag
static MetaGrabberScript FromTag(const QString &tag, bool absolute=false)
Definition: metadatagrabber.cpp:199
MetaGrabberScript::operator=
MetaGrabberScript & operator=(const MetaGrabberScript &other)
Definition: metadatagrabber.cpp:308
MetaGrabberScript::m_author
QString m_author
Definition: metadatagrabber.h:82
MetaGrabberScript::m_description
QString m_description
Definition: metadatagrabber.h:88
gCoreContext
MythCoreContext * gCoreContext
This global variable contains the MythCoreContext instance for the app.
Definition: mythcorecontext.cpp:55
MetaGrabberScript::RunGrabber
MetadataLookupList RunGrabber(const QStringList &args, MetadataLookup *lookup, bool passseas)
Definition: metadatagrabber.cpp:372
MetaGrabberScript::GetGrabber
static MetaGrabberScript GetGrabber(GrabberType defaultType, const MetadataLookup *lookup=nullptr)
Definition: metadatagrabber.cpp:130
kRetagRef
static const QRegularExpression kRetagRef
Definition: metadatagrabber.cpp:24
MythCoreContext::GetLanguage
QString GetLanguage(void)
Returns two character ISO-639 language descriptor for UI language.
Definition: mythcorecontext.cpp:1770
kMSRunShell
@ kMSRunShell
run process through shell
Definition: mythsystem.h:43
MetaGrabberScript::m_version
float m_version
Definition: metadatagrabber.h:90
MetaGrabberScript::FromInetref
static MetaGrabberScript FromInetref(const QString &inetref, bool absolute=false)
Definition: metadatagrabber.cpp:229
grabberTypes
static const QMap< GrabberType, GrabberOpts > grabberTypes
Definition: metadatagrabber.cpp:36
MythDate::secsInPast
std::chrono::seconds secsInPast(const QDateTime &past)
Definition: mythdate.cpp:203
mythcorecontext.h
MetaGrabberScript::m_valid
bool m_valid
Definition: metadatagrabber.h:91
kGrabberInvalid
@ kGrabberInvalid
Definition: metadatagrabber.h:26
kGrabberGame
@ kGrabberGame
Definition: metadatagrabber.h:25
MetadataLookup::GetInetref
QString GetInetref() const
Definition: metadatacommon.h:356
MetaGrabberScript::LookupData
MetadataLookupList LookupData(const QString &inetref, MetadataLookup *lookup, bool passseas=true)
Definition: metadatagrabber.cpp:480
std
Definition: mythchrono.h:23
grabberTypeStrings
static QMap< QString, GrabberType > grabberTypeStrings
Definition: metadatagrabber.cpp:51
MetaGrabberScript::GetType
GrabberType GetType(void) const
Definition: metadatagrabber.h:64
MetaGrabberScript::m_typestring
QString m_typestring
Definition: metadatagrabber.h:87
ParseMetadataItem
MetadataLookup * ParseMetadataItem(const QDomElement &item, MetadataLookup *lookup, bool passseas)
Definition: metadatacommon.cpp:893
s_grabberList
static GrabberList s_grabberList
Definition: metadatagrabber.cpp:26
MetaGrabberScript::MetaGrabberScript
MetaGrabberScript()=default
MythSystemLegacy::Run
void Run(std::chrono::seconds timeout=0s)
Runs a command inside the /bin/sh shell. Returns immediately.
Definition: mythsystemlegacy.cpp:213
GrabberOpts
Definition: metadatagrabber.cpp:30
kGrabberAll
@ kGrabberAll
Definition: metadatagrabber.h:21
exitcodes.h
listLock
static QMutex listLock
Definition: mythsystemunix.cpp:59
kMSStdOut
@ kMSStdOut
allow access to stdout
Definition: mythsystem.h:41
metadatacommon.h
MythCoreContext::GetSetting
QString GetSetting(const QString &key, const QString &defaultval="")
Definition: mythcorecontext.cpp:897