MythTV  master
tedtalks_api.py
Go to the documentation of this file.
1 # -*- coding: UTF-8 -*-
2 
3 # ----------------------
4 # Name: tedtalks_api - Simple-to-use Python interface to the TedTalks RSS feeds
5 # (http://www.ted.com)
6 # Python Script
7 # Author: R.D. Vaughan
8 # Purpose: This python script is intended to perform a variety of utility functions to
9 # search and access text metadata, video and image URLs from TedTalks Web site.
10 #
11 # License:Creative Commons GNU GPL v2
12 # (http://creativecommons.org/licenses/GPL/2.0/)
13 #-------------------------------------
14 __title__ ="tedtalks_api - Simple-to-use Python interface to the TedTalks videos (http://www.ted.com)"
15 __author__="R.D. Vaughan"
16 __purpose__='''
17 This python script is intended to perform a variety of utility functions to search and access text
18 meta data, video and image URLs from the TedTalks Web site. These routines process videos
19 provided by TedTalks (http://www.ted.com). The specific TedTalks RSS feeds that are processed are controled through a user XML preference file usually found at
20 "~/.mythtv/MythNetvision/userGrabberPrefs/tedtalks.xml"
21 '''
22 
23 __version__="v0.1.0"
24 # 0.1.0 Initial development
25 
26 import os, struct, sys, re, time, datetime, shutil, urllib.request, urllib.parse, urllib.error
27 
28 import logging
29 from threading import Thread
30 from copy import deepcopy
31 from operator import itemgetter, attrgetter
32 
33 from .tedtalks_exceptions import (TedTalksUrlError, TedTalksHttpError, TedTalksRssError, TedTalksVideoNotFound, TedTalksConfigFileError, TedTalksUrlDownloadError)
34 import io
35 
36 class OutStreamEncoder(object):
37  """Wraps a stream with an encoder"""
38  def __init__(self, outstream, encoding=None):
39  self.out = outstream
40  if not encoding:
41  self.encoding = sys.getfilesystemencoding()
42  else:
43  self.encoding = encoding
44 
45  def write(self, obj):
46  """Wraps the output stream, encoding Unicode strings with the specified encoding"""
47  if isinstance(obj, str):
48  obj = obj.encode(self.encoding)
49  self.out.buffer.write(obj)
50 
51  def __getattr__(self, attr):
52  """Delegate everything but write to the stream"""
53  return getattr(self.out, attr)
54 
55 if isinstance(sys.stdout, io.TextIOWrapper):
56  sys.stdout = OutStreamEncoder(sys.stdout, 'utf8')
57  sys.stderr = OutStreamEncoder(sys.stderr, 'utf8')
58 
59 
60 try:
61  from io import StringIO
62  from lxml import etree
63 except Exception as e:
64  sys.stderr.write('\n! Error - Importing the "lxml" and "StringIO" python libraries failed on error(%s)\n' % e)
65  sys.exit(1)
66 
67 # Check that the lxml library is current enough
68 # From the lxml documents it states: (http://codespeak.net/lxml/installation.html)
69 # "If you want to use XPath, do not use libxml2 2.6.27. We recommend libxml2 2.7.2 or later"
70 # Testing was performed with the Ubuntu 9.10 "python-lxml" version "2.1.5-1ubuntu2" repository package
71 version = ''
72 for digit in etree.LIBXML_VERSION:
73  version+=str(digit)+'.'
74 version = version[:-1]
75 if version < '2.7.2':
76  sys.stderr.write('''
77 ! Error - The installed version of the "lxml" python library "libxml" version is too old.
78  At least "libxml" version 2.7.2 must be installed. Your version is (%s).
79 ''' % version)
80  sys.exit(1)
81 
82 # Used for debugging
83 #import nv_python_libs.mashups.mashups_api as target
84 try:
85  '''Import the python mashups support classes
86  '''
87  import nv_python_libs.mashups.mashups_api as mashups_api
88 except Exception as e:
89  sys.stderr.write('''
90 The subdirectory "nv_python_libs/mashups" containing the modules mashups_api and
91 mashups_exceptions.py (v0.1.0 or greater),
92 They should have been included with the distribution of tedtalks.py.
93 Error(%s)
94 ''' % e)
95  sys.exit(1)
96 if mashups_api.__version__ < '0.1.0':
97  sys.stderr.write("\n! Error: Your current installed mashups_api.py version is (%s)\nYou must at least have version (0.1.0) or higher.\n" % mashups_api.__version__)
98  sys.exit(1)
99 
100 
101 class Videos(object):
102  """Main interface to http://www.ted.com
103  This is done to support a common naming framework for all python Netvision plugins no matter their
104  site target.
105 
106  Supports search methods
107  The apikey is a not required to access http://www.ted.com
108  """
109  def __init__(self,
110  apikey,
111  mythtv = True,
112  interactive = False,
113  select_first = False,
114  debug = False,
115  custom_ui = None,
116  language = None,
117  search_all_languages = False,
118  ):
119  """apikey (str/unicode):
120  Specify the target site API key. Applications need their own key in some cases
121 
122  mythtv (True/False):
123  When True, the returned meta data is being returned has the key and values massaged to match MythTV
124  When False, the returned meta data is being returned matches what target site returned
125 
126  interactive (True/False): (This option is not supported by all target site apis)
127  When True, uses built-in console UI is used to select the correct show.
128  When False, the first search result is used.
129 
130  select_first (True/False): (This option is not supported currently implemented in any grabbers)
131  Automatically selects the first series search result (rather
132  than showing the user a list of more than one series).
133  Is overridden by interactive = False, or specifying a custom_ui
134 
135  debug (True/False):
136  shows verbose debugging information
137 
138  custom_ui (xx_ui.BaseUI subclass): (This option is not supported currently implemented in any grabbers)
139  A callable subclass of interactive class (overrides interactive option)
140 
141  language (2 character language abbreviation): (This option is not supported by all target site apis)
142  The language of the returned data. Is also the language search
143  uses. Default is "en" (English). For full list, run..
144 
145  search_all_languages (True/False): (This option is not supported by all target site apis)
146  By default, a Netvision grabber will only search in the language specified using
147  the language option. When this is True, it will search for the
148  show in any language
149 
150  """
151  self.config = {}
152 
153  if apikey is not None:
154  self.config['apikey'] = apikey
155  else:
156  pass # TedTalks does not require an apikey
157 
158  self.config['debug_enabled'] = debug # show debugging messages
159  self.common = common
160  self.common.debug = debug # Set the common function debug level
161 
162  self.log_name = 'TedTalks_Grabber'
163  self.common.logger = self.common.initLogger(path=sys.stderr, log_name=self.log_name)
164  self.logger = self.common.logger # Setups the logger (self.log.debug() etc)
165 
166  self.config['custom_ui'] = custom_ui
167 
168  self.config['interactive'] = interactive
169 
170  self.config['select_first'] = select_first
171 
172  self.config['search_all_languages'] = search_all_languages
173 
174  self.error_messages = {'TedTalksUrlError': "! Error: The URL (%s) cause the exception error (%s)\n", 'TedTalksHttpError': "! Error: An HTTP communications error with the TedTalks was raised (%s)\n", 'TedTalksRssError': "! Error: Invalid RSS meta data\nwas received from the TedTalks error (%s). Skipping item.\n", 'TedTalksVideoNotFound': "! Error: Video search with the TedTalks did not return any results (%s)\n", 'TedTalksConfigFileError': "! Error: tedtalks_config.xml file missing\nit should be located in and named as (%s).\n", 'TedTalksUrlDownloadError': "! Error: Downloading a RSS feed or Web page (%s).\n", }
175 
176  # Channel details and search results
177  self.channel = {'channel_title': 'TedTalks', 'channel_link': 'http://www.ted.com', 'channel_description': "TED is a small nonprofit devoted to Ideas Worth Spreading.", 'channel_numresults': 0, 'channel_returned': 1, 'channel_startindex': 0}
178 
179  self.channel_icon = '%SHAREDIR%/mythnetvision/icons/tedtalks.png'
180 
181  self.config['image_extentions'] = ["png", "jpg", "bmp"] # Acceptable image extentions
182 
183  # Initialize Mashups api variables
184  mashups_api.common = self.common
185  self.mashups_api = mashups_api.Videos('')
186  self.mashups_api.channel = self.channel
187  if language:
188  self.mashups_api.config['language'] = self.config['language']
189  self.mashups_api.config['debug_enabled'] = self.config['debug_enabled']
190  self.mashups_api.getUserPreferences = self.getUserPreferences
191  # end __init__()
192 
193 
198 
199  def getTedTalksConfig(self):
200  ''' Read the MNV TedTalks grabber "tedtalks_config.xml" configuration file
201  return nothing
202  '''
203  # Read the grabber tedtalks_config.xml configuration file
204  url = 'file://%s/nv_python_libs/configs/XML/tedtalks_config.xml' % (baseProcessingDir, )
205  if not os.path.isfile(url[7:]):
206  raise TedTalksConfigFileError(self.error_messages['TedTalksConfigFileError'] % (url[7:], ))
207 
208  if self.config['debug_enabled']:
209  print(url)
210  print()
211  try:
212  self.tedtalks_config = etree.parse(url)
213  except Exception as errormsg:
214  raise TedTalksUrlError(self.error_messages['TedTalksUrlError'] % (url, errormsg))
215  return
216  # end getTedTalksConfig()
217 
218 
220  '''Read the tedtalks_config.xml and user preference tedtalks.xml file.
221  If the tedtalks.xml file does not exist then create it.
222  If the tedtalks.xml file is too old then update it.
223  return nothing
224  '''
225  # Get tedtalks_config.xml
226  self.getTedTalksConfig()
227 
228  # Check if the tedtalks.xml file exists
229  userPreferenceFile = self.tedtalks_config.find('userPreferenceFile').text
230  if userPreferenceFile[0] == '~':
231  self.tedtalks_config.find('userPreferenceFile').text = "%s%s" % (os.path.expanduser("~"), userPreferenceFile[1:])
232  if os.path.isfile(self.tedtalks_config.find('userPreferenceFile').text):
233  # Read the grabber tedtalks_config.xml configuration file
234  url = 'file://%s' % (self.tedtalks_config.find('userPreferenceFile').text, )
235  if self.config['debug_enabled']:
236  print(url)
237  print()
238  try:
239  self.userPrefs = etree.parse(url)
240  except Exception as errormsg:
241  raise TedTalksUrlError(self.error_messages['TedTalksUrlError'] % (url, errormsg))
242  create = False
243  else:
244  create = True
245 
246  # If required create/update the tedtalks.xml file
247  self.updateTedTalks(create)
248  return
249  # end getUserPreferences()
250 
251  def updateTedTalks(self, create=False):
252  ''' Create or update the tedtalks.xml user preferences file
253  return nothing
254  '''
255  userDefaultFile = '%s/nv_python_libs/configs/XML/defaultUserPrefs/tedtalks.xml' % (baseProcessingDir, )
256  if os.path.isfile(userDefaultFile):
257  # Read the default tedtalks.xml user preferences file
258  url = 'file://%s' % (userDefaultFile, )
259  if self.config['debug_enabled']:
260  print(url)
261  print()
262  try:
263  userTedTalks = etree.parse(url)
264  except Exception as e:
265  raise TedTalksUrlError(self.error_messages['TedTalksUrlError'] % (url, e))
266  else:
267  raise Exception('!Error: The default TedTalk file is missing (%s)', userDefaultFile)
268 
269  # If there was an existing tedtalks.xml file then add any relevant user settings
270  # to this new tedtalks.xml
271  if not create:
272  for showElement in self.userPrefs.xpath("//sourceURL"):
273  showName = showElement.getparent().attrib['name']
274  sourceName = showElement.attrib['name']
275  elements = userTedTalks.xpath("//sourceURL[@name=$showName]", showName=showName,)
276  if len(elements):
277  elements[0].attrib['enabled'] = showElement.attrib['enabled']
278  elements[0].attrib['parameter'] = showElement.attrib['parameter']
279 
280  if self.config['debug_enabled']:
281  print("After any merging userTedTalks:")
282  sys.stdout.write(etree.tostring(userTedTalks, encoding='UTF-8', pretty_print=True))
283  print()
284 
285  # Save the tedtalks.xml file
286  prefDir = self.tedtalks_config.find('userPreferenceFile').text.replace('/tedtalks.xml', '')
287  if not os.path.isdir(prefDir):
288  os.makedirs(prefDir)
289  fd = open(self.tedtalks_config.find('userPreferenceFile').text, 'w')
290  fd.write(etree.tostring(userTedTalks, encoding='UTF-8', pretty_print=True))
291  fd.close()
292 
293  # Read the refreshed user config file
294  try:
295  self.userPrefs = etree.parse(self.tedtalks_config.find('userPreferenceFile').text)
296  self.mashups_api.userPrefs = self.userPrefs
297  except Exception as errormsg:
298  raise TedTalksUrlError(self.error_messages['TedTalksUrlError'] % (url, errormsg))
299  return
300  # end updateTedTalks()
301 
302 
307 
308  def searchTitle(self, title, pagenumber, pagelen):
309  '''Key word video search of the TedTalks web site
310  return an array of matching item elements
311  return
312  '''
313  searchVar = self.tedtalks_config.find('searchURLS').xpath(".//href")[0].text
314  try:
315  searchVar = searchVar.replace('SEARCHTERM', urllib.parse.quote_plus(title.encode("utf-8")))
316  searchVar = searchVar.replace('PAGENUM', str(pagenumber))
317  except UnicodeDecodeError:
318  searchVar = '?q=%s' % ()
319  searchVar = searchVar.replace('SEARCHTERM', urllib.parse.quote_plus(title))
320  searchVar = searchVar.replace('PAGENUM', str(pagenumber))
321  url = searchVar
322 
323  if self.config['debug_enabled']:
324  print(url)
325  print()
326 
327  self.tedtalks_config.find('searchURLS').xpath(".//href")[0].text = url
328 
329  # Globally add all the xpath extentions to the "mythtv" namespace allowing access within the
330  # XSLT stylesheets
331  self.common.buildFunctionDict()
332  mnvXpath = etree.FunctionNamespace('http://www.mythtv.org/wiki/MythNetvision_Grabber_Script_Format')
333  mnvXpath.prefix = 'mnvXpath'
334  for key in list(self.common.functionDict.keys()):
335  mnvXpath[key] = common.functionDict[key]
336 
337  # Add the parameter element from the User preferences file
338  paraMeter = self.userPrefs.find('search').xpath("//search//sourceURL[@enabled='true']/@parameter")
339  if not len(paraMeter):
340  raise Exception('TedTalks User preferences file "tedtalks.xml" does not have an enabled search with a "parameter" attribute.')
341  etree.SubElement(self.tedtalks_config.find('searchURLS').xpath(".//url")[0], "parameter").text = paraMeter[0]
342 
343  # Perform a search
344  try:
345  resultTree = self.common.getUrlData(self.tedtalks_config.find('searchURLS'))
346  except Exception as errormsg:
347  raise TedTalksUrlDownloadError(self.error_messages['TedTalksUrlDownloadError'] % (errormsg))
348 
349  if resultTree is None:
350  raise TedTalksVideoNotFound("No TedTalks Video matches found for search value (%s)" % title)
351 
352  searchResults = resultTree.xpath('//result//item')
353  if not len(searchResults):
354  raise TedTalksVideoNotFound("No TedTalks Video matches found for search value (%s)" % title)
355 
356  return searchResults
357  # end searchTitle()
358 
359 
360  def searchForVideos(self, title, pagenumber):
361  """Common name for a video search. Used to interface with MythTV plugin NetVision
362  """
363  # Get tedtalks_config.xml
364  self.getUserPreferences()
365 
366  if self.config['debug_enabled']:
367  print("self.tedtalks_config:")
368  sys.stdout.write(etree.tostring(self.tedtalks_config, encoding='UTF-8', pretty_print=True))
369  print()
370 
371  # Easier for debugging
372 # print self.searchTitle(title, pagenumber, self.page_limit)
373 # print
374 # sys.exit()
375 
376  try:
377  data = self.searchTitle(title, pagenumber, self.page_limit)
378  except TedTalksVideoNotFound as msg:
379  sys.stderr.write("%s\n" % msg)
380  sys.exit(0)
381  except TedTalksUrlError as msg:
382  sys.stderr.write('%s\n' % msg)
383  sys.exit(1)
384  except TedTalksHttpError as msg:
385  sys.stderr.write(self.error_messages['TedTalksHttpError'] % msg)
386  sys.exit(1)
387  except TedTalksRssError as msg:
388  sys.stderr.write(self.error_messages['TedTalksRssError'] % msg)
389  sys.exit(1)
390  except Exception as e:
391  sys.stderr.write("! Error: Unknown error during a Video search (%s)\nError(%s)\n" % (title, e))
392  sys.exit(1)
393 
394  # Create RSS element tree
395  rssTree = etree.XML(self.common.mnvRSS+'</rss>')
396 
397  # Set the paging values
398  if len(data) == self.page_limit:
399  self.channel['channel_returned'] = len(data)
400  self.channel['channel_startindex'] = len(data)+(self.page_limit*(int(pagenumber)-1))
401  self.channel['channel_numresults'] = len(data)+(self.page_limit*(int(pagenumber)-1)+1)
402  else:
403  self.channel['channel_returned'] = len(data)+(self.page_limit*(int(pagenumber)-1))
404  self.channel['channel_startindex'] = len(data)
405  self.channel['channel_numresults'] = len(data)
406 
407  # Add the Channel element tree
408  channelTree = self.common.mnvChannelElement(self.channel)
409  rssTree.append(channelTree)
410 
411  for item in data:
412  channelTree.append(item)
413 
414  # Output the MNV search results
415  sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n')
416  sys.stdout.write(etree.tostring(rssTree, encoding='UTF-8', pretty_print=True))
417  sys.exit(0)
418  # end searchForVideos()
419 
420  def displayTreeView(self):
421  '''Gather all videos for each TedTalks show
422  Display the results and exit
423  '''
424  self.mashups_api.page_limit = self.page_limit
425  self.mashups_api.grabber_title = self.grabber_title
426  self.mashups_api.mashup_title = self.mashup_title
427  self.mashups_api.channel_icon = self.channel_icon
428  self.mashups_api.mashup_title = 'tedtalks'
429 
430  # Easier for debugging
431 # self.mashups_api.displayTreeView()
432 # print
433 # sys.exit(1)
434 
435  try:
436  self.mashups_api.Search = False
438  except Exception as e:
439  sys.stderr.write("! Error: During a TedTalks Video treeview\nError(%s)\n" % (e))
440  sys.exit(1)
441 
442  sys.exit(0)
443  # end displayTreeView()
444 # end Videos() class
nv_python_libs.tedtalks.tedtalks_api.Videos.__init__
def __init__(self, apikey, mythtv=True, interactive=False, select_first=False, debug=False, custom_ui=None, language=None, search_all_languages=False)
Definition: tedtalks_api.py:109
nv_python_libs.tedtalks.tedtalks_api.Videos
Definition: tedtalks_api.py:101
nv_python_libs.tedtalks.tedtalks_exceptions.TedTalksUrlDownloadError
Definition: tedtalks_exceptions.py:47
nv_python_libs.tedtalks.tedtalks_exceptions.TedTalksVideoNotFound
Definition: tedtalks_exceptions.py:37
nv_python_libs.tedtalks.tedtalks_api.OutStreamEncoder.__init__
def __init__(self, outstream, encoding=None)
Definition: tedtalks_api.py:38
nv_python_libs.tedtalks.tedtalks_api.Videos.getTedTalksConfig
def getTedTalksConfig(self)
Start - Utility functions.
Definition: tedtalks_api.py:199
nv_python_libs.tedtalks.tedtalks_exceptions.TedTalksUrlError
Definition: tedtalks_exceptions.py:22
nv_python_libs.tedtalks.tedtalks_api.Videos.searchForVideos
def searchForVideos(self, title, pagenumber)
Definition: tedtalks_api.py:360
nv_python_libs.tedtalks.tedtalks_exceptions.TedTalksConfigFileError
Definition: tedtalks_exceptions.py:42
nv_python_libs.tedtalks.tedtalks_api.Videos.channel_icon
channel_icon
Definition: tedtalks_api.py:170
nv_python_libs.tedtalks.tedtalks_api.OutStreamEncoder.encoding
encoding
Definition: tedtalks_api.py:41
nv_python_libs.mashups.mashups_api
Definition: mashups_api.py:1
nv_python_libs.tedtalks.tedtalks_api.Videos.updateTedTalks
def updateTedTalks(self, create=False)
Definition: tedtalks_api.py:251
nv_python_libs.tedtalks.tedtalks_api.Videos.userPrefs
userPrefs
Definition: tedtalks_api.py:239
nv_python_libs.tedtalks.tedtalks_api.OutStreamEncoder.__getattr__
def __getattr__(self, attr)
Definition: tedtalks_api.py:51
nv_python_libs.tedtalks.tedtalks_api.Videos.logger
logger
Definition: tedtalks_api.py:155
nv_python_libs.tedtalks.tedtalks_api.OutStreamEncoder.write
def write(self, obj)
Definition: tedtalks_api.py:45
nv_python_libs.tedtalks.tedtalks_api.Videos.error_messages
error_messages
Definition: tedtalks_api.py:165
nv_python_libs.tedtalks.tedtalks_api.Videos.mashups_api
mashups_api
Definition: tedtalks_api.py:176
print
static void print(const QList< uint > &raw_minimas, const QList< uint > &raw_maximas, const QList< float > &minimas, const QList< float > &maximas)
Definition: vbi608extractor.cpp:29
nv_python_libs.tedtalks.tedtalks_api.Videos.tedtalks_config
tedtalks_config
Definition: tedtalks_api.py:212
nv_python_libs.tedtalks.tedtalks_api.Videos.log_name
log_name
Definition: tedtalks_api.py:153
nv_python_libs.tedtalks.tedtalks_api.Videos.displayTreeView
def displayTreeView(self)
Definition: tedtalks_api.py:420
nv_python_libs.tedtalks.tedtalks_api.Videos.config
config
Definition: tedtalks_api.py:142
nv_python_libs.tedtalks.tedtalks_api.Videos.common
common
Definition: tedtalks_api.py:150
nv_python_libs.tedtalks.tedtalks_api.Videos.getUserPreferences
def getUserPreferences(self)
Definition: tedtalks_api.py:219
find
static pid_list_t::iterator find(const PIDInfoMap &map, pid_list_t &list, pid_list_t::iterator begin, pid_list_t::iterator end, bool find_open)
Definition: dvbstreamhandler.cpp:357
nv_python_libs.tedtalks.tedtalks_api.OutStreamEncoder.out
out
Definition: tedtalks_api.py:39
nv_python_libs.tedtalks.tedtalks_api.Videos.channel
channel
Definition: tedtalks_api.py:168
nv_python_libs.tedtalks.tedtalks_api.OutStreamEncoder
Definition: tedtalks_api.py:36
nv_python_libs.tedtalks.tedtalks_api.Videos.searchTitle
def searchTitle(self, title, pagenumber, pagelen)
End of Utility functions.
Definition: tedtalks_api.py:308