MythTV  master
giantbomb_api.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 # -*- coding: UTF-8 -*-
3 # ----------------------
4 # Name: giantbomb_api.py Simple-to-use Python interface to the GiantBomb's API (api.giantbomb.com)
5 # Python Script
6 # Author: R.D. Vaughan
7 # Purpose: This python script is intended to perform a variety of utility functions to search and
8 # access text metadata and image URLs from GiantBomb. These routines are based on the
9 # GiantBomb api. Specifications for this api are published at:
10 # https://www.giantbomb.com/api/documentation/
11 #
12 # License:Creative Commons GNU GPL v2
13 # (https://creativecommons.org/licenses/GPL/2.0/)
14 #-------------------------------------
15 __title__ ="giantbomb_api - Simple-to-use Python interface to The GiantBomb's API (www.giantbomb.com/api)";
16 __author__="R.D. Vaughan"
17 __purpose__='''
18 This python script is intended to perform a variety of utility functions to search and access text
19 metadata and image URLs from GiantBomb. These routines are based on the GiantBomb api. Specifications
20 for this api are published at https://www.giantbomb.com/api/documentation/
21 '''
22 
23 __version__="v0.2.0"
24 # 0.1.0 Initial development
25 # 0.2.0 R. Ernst: switched to python requests and added python3 compatibility
26 
27 import os, struct, sys, datetime, time, re
28 import requests
29 from copy import deepcopy
30 
31 IS_PY2 = sys.version_info[0] == 2
32 if not IS_PY2:
33  unicode = str
34  unichr = chr
35 
36 from .giantbomb_exceptions import (GiantBombBaseError, GiantBombHttpError, GiantBombXmlError, GiantBombGameNotFound,)
37 
38 
39 class OutStreamEncoder(object):
40  """Wraps a stream with an encoder"""
41  def __init__(self, outstream, encoding=None):
42  self.out = outstream
43  if not encoding:
44  self.encoding = sys.getfilesystemencoding()
45  else:
46  self.encoding = encoding
47 
48  def write(self, obj):
49  """Wraps the output stream, encoding Unicode strings with the specified encoding"""
50  if isinstance(obj, unicode):
51  obj = obj.encode(self.encoding)
52  if IS_PY2:
53  self.out.write(obj)
54  else:
55  self.out.buffer.write(obj)
56 
57  def __getattr__(self, attr):
58  """Delegate everything but write to the stream"""
59  return getattr(self.out, attr)
60 sys.stdout = OutStreamEncoder(sys.stdout, 'utf8')
61 sys.stderr = OutStreamEncoder(sys.stderr, 'utf8')
62 
63 
64 try:
65  if IS_PY2:
66  from StringIO import StringIO
67  else:
68  from io import StringIO
69  from lxml import etree
70 except Exception as e:
71  sys.stderr.write(u'\n! Error - Importing the "lxml" and "StringIO" python libraries failed on error(%s)\n' % e)
72  sys.exit(1)
73 
74 # Check that the lxml library is current enough
75 # From the lxml documents it states: (https://lxml.de/installation.html)
76 # "If you want to use XPath, do not use libxml2 2.6.27. We recommend libxml2 2.7.2 or later"
77 # Testing was performed with the Ubuntu 9.10 "python-lxml" version "2.1.5-1ubuntu2" repository package
78 version = ''
79 for digit in etree.LIBXML_VERSION:
80  version+=str(digit)+'.'
81 version = version[:-1]
82 if version < '2.7.2':
83  sys.stderr.write(u'''
84 ! Error - The installed version of the "lxml" python library "libxml" version is too old.
85  At least "libxml" version 2.7.2 must be installed. Your version is (%s).
86 ''' % version)
87  sys.exit(1)
88 
89 
90 class gamedbQueries():
91  '''Methods that query api.giantbomb.com for metadata and outputs the results to stdout any errors are output
92  to stderr.
93  '''
94  def __init__(self,
95  apikey,
96  debug = False,
97  ):
98  """apikey (str/unicode):
99  Specify the api.giantbomb.com API key. Applications need their own key.
100  See https://www.giantbomb.com/api/ to get your own API key
101 
102  debug (True/False):
103  shows verbose debugging information
104  """
105  self.config = {}
106 
107  self.config['apikey'] = apikey
108  self.config['debug'] = debug
109  self.config['searchURL'] = u'https://www.giantbomb.com/api/search'
110  self.config['dataURL'] = u'https://www.giantbomb.com/api/game/%s'
111  # giantbomb.com now requires a unique 'User-Agent':
112  self.config['headers'] = {"User-Agent": 'MythTV giantbomb grabber 0.2'}
113 
114  self.error_messages = {'GiantBombHttpError': u"! Error: A connection error to api.giantbomb.com was raised (%s)\n", 'GiantBombXmlError': u"! Error: Invalid XML was received from api.giantbomb.com (%s)\n", 'GiantBombBaseError': u"! Error: An error was raised (%s)\n", }
115 
116  self.baseProcessingDir = os.path.dirname( os.path.realpath( __file__ ))
117 
118  self.pubDateFormat = u'%a, %d %b %Y %H:%M:%S GMT'
119  self.xmlParser = etree.XMLParser(remove_blank_text=True)
120 
121  self.supportedJobList = ["actor", "author", "producer", "executive producer", "director", "cinematographer", "composer", "editor", "casting", "voice actor", "music", "writer", "technical director", "design director", ]
122  self.tagTranslations = {
123  'actor': 'Actor',
124  'author': 'Author',
125  'producer': 'Producer',
126  'executive producer': 'Executive Producer',
127  'director': 'Director',
128  'cinematographer': 'Cinematographer',
129  'composer': 'Composer',
130  'editor': 'Editor',
131  'casting': 'Casting',
132  'voice actor': 'Actor',
133  'music': 'Composer',
134  'writer': 'Author',
135  'technical director': 'Director',
136  'design director': 'Director',
137  }
138  # end __init__()
139 
140 
141  def massageText(self, text):
142  '''Removes HTML markup from a text string.
143  @param text The HTML source.
144  @return The plain text. If the HTML source contains non-ASCII
145  entities or character references, this is a Unicode string.
146  '''
147  def fixup(m):
148  text = m.group(0)
149  if text[:1] == "<":
150  return "" # ignore tags
151  if text[:2] == "&#":
152  try:
153  if text[:3] == "&#x":
154  return unichr(int(text[3:-1], 16))
155  else:
156  return unichr(int(text[2:-1]))
157  except ValueError:
158  pass
159  elif text[:1] == "&":
160  if IS_PY2:
161  from htmlentitydefs import entitydefs
162  else:
163  from html.entities import entitydefs
164  entity = entitydefs.get(text[1:-1])
165  if entity:
166  if entity[:2] == "&#":
167  try:
168  return unichr(int(entity[2:-1]))
169  except ValueError:
170  pass
171  else:
172  return unicode(entity, "iso-8859-1")
173  return text # leave as is
174  if IS_PY2:
175  text23 = self.ampReplace(re.sub(u"(?s)<[^>]*>|&#?\w+;", fixup, self.textUtf8(text))).replace(u'\n',u' ')
176  else:
177  text23 = self.ampReplace(re.sub(r"(?s)<[^>]*>|&#?\w+;", fixup, self.textUtf8(text))).replace('\n',' ')
178  return text23
179  # end massageText()
180 
181 
182  def textUtf8(self, text):
183  if text is None:
184  return text
185  try:
186  return unicode(text, 'utf8')
187  except UnicodeDecodeError:
188  return u''
189  except (UnicodeEncodeError, TypeError):
190  return text
191  # end textUtf8()
192 
193 
194  def ampReplace(self, text):
195  '''Replace all "&" characters with "&amp;"
196  '''
197  text = self.textUtf8(text)
198  return text.replace(u'&amp;',u'~~~~~').replace(u'&',u'&amp;').replace(u'~~~~~', u'&amp;')
199  # end ampReplace()
200 
201 
202  def htmlToString(self, context, html):
203  ''' Remove HTML tags and LFs from a string
204  return the string without HTML tags or LFs
205  '''
206  if not len(html):
207  return u""
208  return self.massageText(html).strip().replace(u'\n', u' ').replace(u'’', u"&apos;").replace(u'“', u"&apos;")
209  # end htmlToString()
210 
211  def getHtmlData(self, context, *args):
212  ''' Take a HTML string and convert it to an HTML element. Then apply a filter and return
213  the results.
214  return filter array
215  return an empty array if the filter failed to find any values.
216  '''
217  xpathFilter = None
218  if len(args) > 1:
219  xpathFilter = args[0]
220  htmldata = args[1]
221  else:
222  htmldata = args[0]
223  if not htmldata:
224  return []
225  htmlElement = etree.HTML(htmldata)
226  if not xpathFilter:
227  return htmlElement
228  filteredData = htmlElement.xpath(xpathFilter)
229  if len(filteredData):
230  if xpathFilter.find('@') != -1:
231  return filteredData[0]
232  else:
233  return filteredData[0].text
234  return u''
235  # end getHtmlData()
236 
237  def pubDate(self, context, *inputArgs):
238  '''Convert a date/time string in a specified format into a pubDate. The default is the
239  MNV item format
240  return the formatted pubDate string
241  return on error return the original date string
242  '''
243  args = []
244  for arg in inputArgs:
245  args.append(arg)
246  if args[0] == u'':
247  return datetime.datetime.now().strftime(self.pubDateFormat)
248  index = args[0].find('+')
249  if index == -1:
250  index = args[0].find('-')
251  if index != -1 and index > 5:
252  args[0] = args[0][:index].strip()
253  args[0] = args[0].replace(',', u'').replace('.', u'')
254  try:
255  if len(args) > 2:
256  pubdate = time.strptime(args[0], args[2])
257  elif len(args) > 1:
258  args[1] = args[1].replace(',', u'').replace('.', u'')
259  if args[1].find('GMT') != -1:
260  args[1] = args[1][:args[1].find('GMT')].strip()
261  args[0] = args[0][:args[0].rfind(' ')].strip()
262  try:
263  pubdate = time.strptime(args[0], args[1])
264  except ValueError:
265  if args[1] == '%a %d %b %Y %H:%M:%S':
266  pubdate = time.strptime(args[0], '%a %d %B %Y %H:%M:%S')
267  elif args[1] == '%a %d %B %Y %H:%M:%S':
268  pubdate = time.strptime(args[0], '%a %d %b %Y %H:%M:%S')
269  if len(args) > 2:
270  return time.strftime(args[2], pubdate)
271  else:
272  return time.strftime(self.pubDateFormat, pubdate)
273  else:
274  return datetime.datetime.now().strftime(self.pubDateFormat)
275  except Exception as err:
276  sys.stderr.write(u'! Error: pubDate variables(%s) error(%s)\n' % (args, err))
277  return args[0]
278  # end pubDate()
279 
280  def futureReleaseDate(self, context, gameElement):
281  '''Convert the "expected" release date into the default MNV item format.
282  return the formatted pubDate string
283  return If there is not enough information to make a date then return an empty string
284  '''
285  try:
286  if gameElement.find('expected_release_year').text is not None:
287  year = gameElement.find('expected_release_year').text
288  else:
289  year = None
290  if gameElement.find('expected_release_quarter').text is not None:
291  quarter = gameElement.find('expected_release_quarter').text
292  else:
293  quarter = None
294  if gameElement.find('expected_release_month').text is not None:
295  month = gameElement.find('expected_release_month').text
296  else:
297  month = None
298  except:
299  return u''
300  if not year:
301  return u''
302  if month and not quarter:
303  pubdate = time.strptime((u'%s-%s-01' % (year, month)), '%Y-%m-%d')
304  elif not month and quarter:
305  month = str((int(quarter)*3))
306  pubdate = time.strptime((u'%s-%s-01' % (year, month)), '%Y-%m-%d')
307  else:
308  pubdate = time.strptime((u'%s-12-01' % (year, )), '%Y-%m-%d')
309 
310  return time.strftime('%Y-%m-%d', pubdate)
311  # end futureReleaseDate()
312 
313  def findImages(self, context, *args):
314  '''Parse the "image" and "description" elements for images and put in a persistant array
315  return True when there are images available
316  return False if there are no images
317  '''
318  def makeImageElement(typeImage, url, thumb):
319  ''' Create a single Image element
320  return the image element
321  '''
322  imageElement = etree.XML(u"<image></image>")
323  imageElement.attrib['type'] = typeImage
324  imageElement.attrib['url'] = url
325  imageElement.attrib['thumb'] = thumb
326  return imageElement
327  # end makeImageElement()
328 
329  superImageFilter = etree.XPath('.//super_url/text()')
330  self.imageElements = []
331  for imageElement in args[0]:
332  imageList = superImageFilter(imageElement)
333  if len(imageList):
334  for image in imageList:
335  self.imageElements.append(makeImageElement('coverart', image, image.replace(u'super', u'thumb')))
336  htmlElement = self.getHtmlData('dummy', etree.tostring(args[1][0], method="text", encoding=unicode).strip())
337  if len(htmlElement):
338  for image in htmlElement[0].xpath('.//a/img/@src'):
339  if image.find('screen') == -1:
340  continue
341  if image.find('thumb') == -1:
342  continue
343  self.imageElements.append(makeImageElement('screenshot', image.replace(u'thumb', u'super'), image))
344 
345  if len(args) > 2:
346  for imageElement in args[2]:
347  imageList = superImageFilter(imageElement)
348  if len(imageList):
349  for image in imageList:
350  self.imageElements.append(makeImageElement('screenshot', image, image.replace(u'super', u'thumb')))
351 
352  if not len(self.imageElements):
353  return False
354  return True
355  # end findImages()
356 
357  def getImages(self, context, arg):
358  '''Return an array of image elements that was created be a previous "findImages" function call
359  return the array of image elements
360  '''
361  return self.imageElements
362  # end getImages()
363 
364  def supportedJobs(self, context, *inputArgs):
365  '''Validate that the job category is supported by the
366  Universal Metadata Format item format
367  return True is supported
368  return False if not supported
369  '''
370  if type([]) == type(inputArgs[0]):
371  tmpCopy = inputArgs[0]
372  else:
373  tmpCopy = [inputArgs[0]]
374  for job in tmpCopy:
375  if job.lower() in self.supportedJobList:
376  return True
377  return False
378  # end supportedJobs()
379 
380  def translateName(self, context, *inputArgs):
381  '''Translate a tag name into the Universal Metadata Format item equivalent
382  return the translated tag equivalent
383  return the input name as the name does not need translating and is already been validated
384  '''
385  name = inputArgs[0]
386  name = name.lower()
387  if name in list(self.tagTranslations.keys()):
388  return self.tagTranslations[name]
389  return name
390  # end translateName()
391 
392  def buildFuncDict(self):
393  """ Build a dictionary of the XPath extention function for the XSLT stylesheets
394  Returns nothing
395  """
396  self.FuncDict = {
397  'htmlToString': self.htmlToString,
398  'getHtmlData': self.getHtmlData,
399  'pubDate': self.pubDate,
400  'futureReleaseDate': self.futureReleaseDate,
401  'findImages': self.findImages,
402  'getImages': self.getImages,
403  'supportedJobs': self.supportedJobs,
404  'translateName': self.translateName,
405  }
406  return
407  # end buildFuncDict()
408 
409  def gameSearch(self, gameTitle):
410  """Display a Game query in XML format:
411  https://www.mythtv.org/wiki/MythTV_Universal_Metadata_Format
412  Returns nothing
413  """
414  with requests.Session() as ReqSession:
415  url = self.config['searchURL']
416 
417  params = {}
418  params["api_key"] = self.config['apikey']
419  params["format"] = 'xml'
420  params["page"] = 0
421  params["query"] = gameTitle
422  params["resources"] = "game"
423 
424  headers = self.config['headers']
425 
426  res = ReqSession.get(url, params=params, headers=headers)
427 
428  try:
429  queryResult = etree.fromstring(res.content)
430  except Exception as errmsg:
431  sys.stderr.write(u"! Error: Invalid XML was received from www.giantbomb.com (%s)\n" % errmsg)
432  sys.exit(1)
433 
434  queryXslt = etree.XSLT(etree.parse(u'%s/XSLT/giantbombQuery.xsl' % self.baseProcessingDir))
435  gamebombXpath = etree.FunctionNamespace('https://www.mythtv.org/wiki/MythTV_Universal_Metadata_Format')
436  gamebombXpath.prefix = 'gamebombXpath'
437  self.buildFuncDict()
438  for key in list(self.FuncDict.keys()):
439  gamebombXpath[key] = self.FuncDict[key]
440 
441  items = queryXslt(queryResult)
442 
443  if items.getroot() is not None:
444  if len(items.xpath('//item')):
445  sys.stdout.write(etree.tostring(items, encoding='UTF-8', method="xml", xml_declaration=True, pretty_print=True, ))
446  sys.exit(0)
447  # end gameSearch()
448 
449  def gameData(self, gameId):
450  """Display a Game details in XML format:
451  https://www.mythtv.org/wiki/MythTV_Universal_Metadata_Format
452  Returns nothing
453  """
454  with requests.Session() as ReqSession:
455  url = self.config['dataURL'] % gameId
456 
457  params = {}
458  params["api_key"] = self.config['apikey']
459  params["format"] = 'xml'
460 
461  headers = self.config['headers']
462 
463  res = ReqSession.get(url, params=params, headers=headers)
464 
465  try:
466  videoResult = etree.fromstring(res.content)
467  except Exception as errmsg:
468  sys.stderr.write(u"! Error: Invalid XML was received from www.giantbomb.com (%s)\n" % errmsg)
469  sys.exit(1)
470 
471  gameXslt = etree.XSLT(etree.parse(u'%s/XSLT/giantbombGame.xsl' % self.baseProcessingDir))
472  gamebombXpath = etree.FunctionNamespace('https://www.mythtv.org/wiki/MythTV_Universal_Metadata_Format')
473  gamebombXpath.prefix = 'gamebombXpath'
474  self.buildFuncDict()
475  for key in list(self.FuncDict.keys()):
476  gamebombXpath[key] = self.FuncDict[key]
477  items = gameXslt(videoResult)
478 
479  if items.getroot() is not None:
480  if len(items.xpath('//item')):
481  sys.stdout.write(etree.tostring(items, encoding='UTF-8', method="xml", xml_declaration=True, pretty_print=True, ))
482  sys.exit(0)
483  # end gameData()
484 
485 # end Class gamedbQueries()
486 
487 def main():
488  """Simple example of using giantbomb_api - it just
489  searches for any Game with the word "Grand" in its title and returns a list of matches
490  in Universal XML format. Also gets game details using a GameBomb#.
491  """
492  # api.giantbomb.com api key provided for MythTV
493  api_key = "b5883a902a8ed88b15ce21d07787c94fd6ad9f33"
494  gamebomb = gamedbQueries(api_key)
495  # Output a dictionary of matching movie titles
496  gamebomb.gameSearch(u'Grand')
497  print()
498  # Output a dictionary of matching movie details for GiantBomb number '19995'
499  gamebomb.gameData(u'19995')
500 # end main()
501 
502 if __name__ == '__main__':
503  main()
giantbomb.giantbomb_api.gamedbQueries.pubDateFormat
pubDateFormat
Definition: giantbomb_api.py:115
giantbomb.giantbomb_api.gamedbQueries.config
config
Definition: giantbomb_api.py:102
giantbomb.giantbomb_api.gamedbQueries.imageElements
imageElements
Definition: giantbomb_api.py:330
giantbomb.giantbomb_api.gamedbQueries.pubDate
def pubDate(self, context, *inputArgs)
Definition: giantbomb_api.py:237
giantbomb.giantbomb_api.gamedbQueries.findImages
def findImages(self, context, *args)
Definition: giantbomb_api.py:313
giantbomb.giantbomb_api.gamedbQueries.futureReleaseDate
def futureReleaseDate(self, context, gameElement)
Definition: giantbomb_api.py:280
giantbomb.giantbomb_api.gamedbQueries.tagTranslations
tagTranslations
Definition: giantbomb_api.py:119
giantbomb.giantbomb_api.gamedbQueries.translateName
def translateName(self, context, *inputArgs)
Definition: giantbomb_api.py:380
giantbomb.giantbomb_api.gamedbQueries.xmlParser
xmlParser
Definition: giantbomb_api.py:116
giantbomb.giantbomb_api.OutStreamEncoder
Definition: giantbomb_api.py:39
giantbomb.giantbomb_api.gamedbQueries.buildFuncDict
def buildFuncDict(self)
Definition: giantbomb_api.py:392
giantbomb.giantbomb_api.gamedbQueries.supportedJobs
def supportedJobs(self, context, *inputArgs)
Definition: giantbomb_api.py:364
giantbomb.giantbomb_api.gamedbQueries.textUtf8
def textUtf8(self, text)
Definition: giantbomb_api.py:182
giantbomb.giantbomb_api.OutStreamEncoder.write
def write(self, obj)
Definition: giantbomb_api.py:48
giantbomb.giantbomb_api.OutStreamEncoder.encoding
encoding
Definition: giantbomb_api.py:44
giantbomb.giantbomb_api.gamedbQueries.htmlToString
def htmlToString(self, context, html)
Definition: giantbomb_api.py:202
giantbomb.giantbomb_api.main
def main()
Definition: giantbomb_api.py:487
giantbomb.giantbomb_api.gamedbQueries.getHtmlData
def getHtmlData(self, context, *args)
Definition: giantbomb_api.py:211
giantbomb.giantbomb_api.gamedbQueries.FuncDict
FuncDict
Definition: giantbomb_api.py:396
giantbomb.giantbomb_api.gamedbQueries.gameSearch
def gameSearch(self, gameTitle)
Definition: giantbomb_api.py:409
print
static void print(const QList< uint > &raw_minimas, const QList< uint > &raw_maximas, const QList< float > &minimas, const QList< float > &maximas)
Definition: vbi608extractor.cpp:29
giantbomb.giantbomb_api.OutStreamEncoder.__init__
def __init__(self, outstream, encoding=None)
Definition: giantbomb_api.py:41
giantbomb.giantbomb_api.gamedbQueries.supportedJobList
supportedJobList
Definition: giantbomb_api.py:118
giantbomb.giantbomb_api.gamedbQueries.__init__
def __init__(self, apikey, debug=False)
Definition: giantbomb_api.py:94
giantbomb.giantbomb_api.unichr
unichr
Definition: giantbomb_api.py:34
giantbomb.giantbomb_api.gamedbQueries
Definition: giantbomb_api.py:90
giantbomb.giantbomb_api.gamedbQueries.error_messages
error_messages
Definition: giantbomb_api.py:111
giantbomb.giantbomb_api.gamedbQueries.massageText
def massageText(self, text)
Definition: giantbomb_api.py:141
giantbomb.giantbomb_api.gamedbQueries.ampReplace
def ampReplace(self, text)
Definition: giantbomb_api.py:194
giantbomb.giantbomb_api.OutStreamEncoder.__getattr__
def __getattr__(self, attr)
Definition: giantbomb_api.py:57
giantbomb.giantbomb_api.unicode
unicode
Definition: giantbomb_api.py:33
giantbomb.giantbomb_api.gamedbQueries.baseProcessingDir
baseProcessingDir
Definition: giantbomb_api.py:113
giantbomb.giantbomb_api.gamedbQueries.gameData
def gameData(self, gameId)
Definition: giantbomb_api.py:449
find
static pid_list_t::iterator find(const PIDInfoMap &map, pid_list_t &list, pid_list_t::iterator begin, pid_list_t::iterator end, bool find_open)
Definition: dvbstreamhandler.cpp:357
giantbomb.giantbomb_api.gamedbQueries.getImages
def getImages(self, context, arg)
Definition: giantbomb_api.py:357
giantbomb.giantbomb_api.OutStreamEncoder.out
out
Definition: giantbomb_api.py:42