tedtalks_api.py

Go to the documentation of this file.

# -*- coding: UTF-8 -*-
 
# ----------------------
# Name: tedtalks_api - Simple-to-use Python interface to the TedTalks RSS feeds
#                       (http://www.ted.com)
# Python Script
# Author:   R.D. Vaughan
# Purpose:  This python script is intended to perform a variety of utility functions to
#           search and access text metadata, video and image URLs from TedTalks Web site.
#
# License:Creative Commons GNU GPL v2
# (http://creativecommons.org/licenses/GPL/2.0/)
#-------------------------------------
__title__ ="tedtalks_api - Simple-to-use Python interface to the TedTalks videos (http://www.ted.com)"
__author__="R.D. Vaughan"
__purpose__='''
This python script is intended to perform a variety of utility functions to search and access text
meta data, video and image URLs from the TedTalks Web site. These routines process videos
provided by TedTalks (http://www.ted.com). The specific TedTalks RSS feeds that are processed are controled through a user XML preference file usually found at
"~/.mythtv/MythNetvision/userGrabberPrefs/tedtalks.xml"
'''
 
__version__="v0.1.0"
# 0.1.0 Initial development
 
import os, struct, sys, re, time, datetime, shutil, urllib.request, urllib.parse, urllib.error
 
import logging
from threading import Thread
from copy import deepcopy
from operator import itemgetter, attrgetter
 
from .tedtalks_exceptions import (TedTalksUrlError, TedTalksHttpError, TedTalksRssError, TedTalksVideoNotFound, TedTalksConfigFileError, TedTalksUrlDownloadError)
import io
 
class OutStreamEncoder(object):
    """Wraps a stream with an encoder"""
    def __init__(self, outstream, encoding=None):
        self.out = outstream
        if not encoding:
            self.encoding = sys.getfilesystemencoding()
        else:
            self.encoding = encoding
 
    def write(self, obj):
        """Wraps the output stream, encoding Unicode strings with the specified encoding"""
        if isinstance(obj, str):
            obj = obj.encode(self.encoding)
        self.out.buffer.write(obj)
 
    def __getattr__(self, attr):
        """Delegate everything but write to the stream"""
        return getattr(self.out, attr)
 
if isinstance(sys.stdout, io.TextIOWrapper):
    sys.stdout = OutStreamEncoder(sys.stdout, 'utf8')
    sys.stderr = OutStreamEncoder(sys.stderr, 'utf8')
 
 
try:
    from io import StringIO
    from lxml import etree
except Exception as e:
    sys.stderr.write('\n! Error - Importing the "lxml" and "StringIO" python libraries failed on error(%s)\n' % e)
    sys.exit(1)
 
# Check that the lxml library is current enough
# From the lxml documents it states: (http://codespeak.net/lxml/installation.html)
# "If you want to use XPath, do not use libxml2 2.6.27. We recommend libxml2 2.7.2 or later"
# Testing was performed with the Ubuntu 9.10 "python-lxml" version "2.1.5-1ubuntu2" repository package
version = ''
for digit in etree.LIBXML_VERSION:
    version+=str(digit)+'.'
version = version[:-1]
if version < '2.7.2':
    sys.stderr.write('''
! Error - The installed version of the "lxml" python library "libxml" version is too old.
          At least "libxml" version 2.7.2 must be installed. Your version is (%s).
''' % version)
    sys.exit(1)
 
# Used for debugging
#import nv_python_libs.mashups.mashups_api as target
try:
    '''Import the python mashups support classes
    '''
    import nv_python_libs.mashups.mashups_api as mashups_api
except Exception as e:
    sys.stderr.write('''
The subdirectory "nv_python_libs/mashups" containing the modules mashups_api and
mashups_exceptions.py (v0.1.0 or greater),
They should have been included with the distribution of tedtalks.py.
Error(%s)
''' % e)
    sys.exit(1)
if mashups_api.__version__ < '0.1.0':
    sys.stderr.write("\n! Error: Your current installed mashups_api.py version is (%s)\nYou must at least have version (0.1.0) or higher.\n" % mashups_api.__version__)
    sys.exit(1)
 
 
class Videos(object):
    """Main interface to http://www.ted.com
    This is done to support a common naming framework for all python Netvision plugins no matter their
    site target.
 
    Supports search methods
    The apikey is a not required to access http://www.ted.com
    """
    def __init__(self,
                apikey,
                mythtv = True,
                interactive = False,
                select_first = False,
                debug = False,
                custom_ui = None,
                language = None,
                search_all_languages = False,
                ):
        """apikey (str/unicode):
            Specify the target site API key. Applications need their own key in some cases
 
        mythtv (True/False):
            When True, the returned meta data is being returned has the key and values massaged to match MythTV
            When False, the returned meta data  is being returned matches what target site returned
 
        interactive (True/False): (This option is not supported by all target site apis)
            When True, uses built-in console UI is used to select the correct show.
            When False, the first search result is used.
 
        select_first (True/False): (This option is not supported currently implemented in any grabbers)
            Automatically selects the first series search result (rather
            than showing the user a list of more than one series).
            Is overridden by interactive = False, or specifying a custom_ui
 
        debug (True/False):
             shows verbose debugging information
 
        custom_ui (xx_ui.BaseUI subclass): (This option is not supported currently implemented in any grabbers)
            A callable subclass of interactive class (overrides interactive option)
 
        language (2 character language abbreviation): (This option is not supported by all target site apis)
            The language of the returned data. Is also the language search
            uses. Default is "en" (English). For full list, run..
 
        search_all_languages (True/False): (This option is not supported by all target site apis)
            By default, a Netvision grabber will only search in the language specified using
            the language option. When this is True, it will search for the
            show in any language
 
        """
        self.config = {}
 
        if apikey is not None:
            self.config['apikey'] = apikey
        else:
            pass    # TedTalks does not require an apikey
 
        self.config['debug_enabled'] = debug # show debugging messages
        self.common = common
        self.common.debug = debug   # Set the common function debug level
 
        self.log_name = 'TedTalks_Grabber'
        self.common.logger = self.common.initLogger(path=sys.stderr, log_name=self.log_name)
        self.logger = self.common.logger # Setups the logger (self.log.debug() etc)
 
        self.config['custom_ui'] = custom_ui
 
        self.config['interactive'] = interactive
 
        self.config['select_first'] = select_first
 
        self.config['search_all_languages'] = search_all_languages
 
        self.error_messages = {'TedTalksUrlError': "! Error: The URL (%s) cause the exception error (%s)\n", 'TedTalksHttpError': "! Error: An HTTP communications error with the TedTalks was raised (%s)\n", 'TedTalksRssError': "! Error: Invalid RSS meta data\nwas received from the TedTalks error (%s). Skipping item.\n", 'TedTalksVideoNotFound': "! Error: Video search with the TedTalks did not return any results (%s)\n", 'TedTalksConfigFileError': "! Error: tedtalks_config.xml file missing\nit should be located in and named as (%s).\n", 'TedTalksUrlDownloadError': "! Error: Downloading a RSS feed or Web page (%s).\n", }
 
        # Channel details and search results
        self.channel = {'channel_title': 'TedTalks', 'channel_link': 'http://www.ted.com', 'channel_description': "TED is a small nonprofit devoted to Ideas Worth Spreading.", 'channel_numresults': 0, 'channel_returned': 1, 'channel_startindex': 0}
 
        self.channel_icon = '%SHAREDIR%/mythnetvision/icons/tedtalks.png'
 
        self.config['image_extentions'] = ["png", "jpg", "bmp"] # Acceptable image extentions
 
        # Initialize Mashups api variables
        mashups_api.common = self.common
        self.mashups_api = mashups_api.Videos('')
        self.mashups_api.channel = self.channel
        if language:
            self.mashups_api.config['language'] = self.config['language']
        self.mashups_api.config['debug_enabled'] = self.config['debug_enabled']
        self.mashups_api.getUserPreferences = self.getUserPreferences
    # end __init__()
 
 
 
    def getTedTalksConfig(self):
        ''' Read the MNV TedTalks grabber "tedtalks_config.xml" configuration file
        return nothing
        '''
        # Read the grabber tedtalks_config.xml configuration file
        url = 'file://%s/nv_python_libs/configs/XML/tedtalks_config.xml' % (baseProcessingDir, )
        if not os.path.isfile(url[7:]):
            raise TedTalksConfigFileError(self.error_messages['TedTalksConfigFileError'] % (url[7:], ))
 
        if self.config['debug_enabled']:
            print(url)
            print()
        try:
            self.tedtalks_config = etree.parse(url)
        except Exception as errormsg:
            raise TedTalksUrlError(self.error_messages['TedTalksUrlError'] % (url, errormsg))
        return
    # end getTedTalksConfig()
 
 
    def getUserPreferences(self):
        '''Read the tedtalks_config.xml and user preference tedtalks.xml file.
        If the tedtalks.xml file does not exist then create it.
        If the tedtalks.xml file is too old then update it.
        return nothing
        '''
        # Get tedtalks_config.xml
        self.getTedTalksConfig()
 
        # Check if the tedtalks.xml file exists
        userPreferenceFile = self.tedtalks_config.find('userPreferenceFile').text
        if userPreferenceFile[0] == '~':
             self.tedtalks_config.find('userPreferenceFile').text = "%s%s" % (os.path.expanduser("~"), userPreferenceFile[1:])
        if os.path.isfile(self.tedtalks_config.find('userPreferenceFile').text):
            # Read the grabber tedtalks_config.xml configuration file
            url = 'file://%s' % (self.tedtalks_config.find('userPreferenceFile').text, )
            if self.config['debug_enabled']:
                print(url)
                print()
            try:
                self.userPrefs = etree.parse(url)
            except Exception as errormsg:
                raise TedTalksUrlError(self.error_messages['TedTalksUrlError'] % (url, errormsg))
            create = False
        else:
            create = True
 
        # If required create/update the tedtalks.xml file
        self.updateTedTalks(create)
        return
    # end getUserPreferences()
 
    def updateTedTalks(self, create=False):
        ''' Create or update the tedtalks.xml user preferences file
        return nothing
        '''
        userDefaultFile = '%s/nv_python_libs/configs/XML/defaultUserPrefs/tedtalks.xml' % (baseProcessingDir, )
        if os.path.isfile(userDefaultFile):
            # Read the default tedtalks.xml user preferences file
            url = 'file://%s' % (userDefaultFile, )
            if self.config['debug_enabled']:
                print(url)
                print()
            try:
                userTedTalks = etree.parse(url)
            except Exception as e:
                raise TedTalksUrlError(self.error_messages['TedTalksUrlError'] % (url, e))
        else:
            raise Exception('!Error: The default TedTalk file is missing (%s)', userDefaultFile)
 
        # If there was an existing tedtalks.xml file then add any relevant user settings
        # to this new tedtalks.xml
        if not create:
            for showElement in self.userPrefs.xpath("//sourceURL"):
                showName = showElement.getparent().attrib['name']
                sourceName = showElement.attrib['name']
                elements = userTedTalks.xpath("//sourceURL[@name=$showName]", showName=showName,)
                if len(elements):
                    elements[0].attrib['enabled'] = showElement.attrib['enabled']
                    elements[0].attrib['parameter'] = showElement.attrib['parameter']
 
        if self.config['debug_enabled']:
            print("After any merging userTedTalks:")
            sys.stdout.write(etree.tostring(userTedTalks, encoding='UTF-8', pretty_print=True))
            print()
 
        # Save the tedtalks.xml file
        prefDir = self.tedtalks_config.find('userPreferenceFile').text.replace('/tedtalks.xml', '')
        if not os.path.isdir(prefDir):
            os.makedirs(prefDir)
        fd = open(self.tedtalks_config.find('userPreferenceFile').text, 'w')
        fd.write(etree.tostring(userTedTalks, encoding='UTF-8', pretty_print=True))
        fd.close()
 
        # Read the refreshed user config file
        try:
            self.userPrefs = etree.parse(self.tedtalks_config.find('userPreferenceFile').text)
            self.mashups_api.userPrefs = self.userPrefs
        except Exception as errormsg:
            raise TedTalksUrlError(self.error_messages['TedTalksUrlError'] % (url, errormsg))
        return
    # end updateTedTalks()
 
 
 
    def searchTitle(self, title, pagenumber, pagelen):
        '''Key word video search of the TedTalks web site
        return an array of matching item elements
        return
        '''
        searchVar = self.tedtalks_config.find('searchURLS').xpath(".//href")[0].text
        try:
            searchVar = searchVar.replace('SEARCHTERM', urllib.parse.quote_plus(title.encode("utf-8")))
            searchVar = searchVar.replace('PAGENUM', str(pagenumber))
        except UnicodeDecodeError:
            searchVar = '?q=%s' % ()
            searchVar = searchVar.replace('SEARCHTERM', urllib.parse.quote_plus(title))
            searchVar = searchVar.replace('PAGENUM', str(pagenumber))
        url = searchVar
 
        if self.config['debug_enabled']:
            print(url)
            print()
 
        self.tedtalks_config.find('searchURLS').xpath(".//href")[0].text = url
 
        # Globally add all the xpath extentions to the "mythtv" namespace allowing access within the
        # XSLT stylesheets
        self.common.buildFunctionDict()
        mnvXpath = etree.FunctionNamespace('http://www.mythtv.org/wiki/MythNetvision_Grabber_Script_Format')
        mnvXpath.prefix = 'mnvXpath'
        for key in list(self.common.functionDict.keys()):
            mnvXpath[key] = common.functionDict[key]
 
        # Add the parameter element from the User preferences file
        paraMeter = self.userPrefs.find('search').xpath("//search//sourceURL[@enabled='true']/@parameter")
        if not len(paraMeter):
            raise Exception('TedTalks User preferences file "tedtalks.xml" does not have an enabled search with a "parameter" attribute.')
        etree.SubElement(self.tedtalks_config.find('searchURLS').xpath(".//url")[0], "parameter").text = paraMeter[0]
 
        # Perform a search
        try:
            resultTree = self.common.getUrlData(self.tedtalks_config.find('searchURLS'))
        except Exception as errormsg:
            raise TedTalksUrlDownloadError(self.error_messages['TedTalksUrlDownloadError'] % (errormsg))
 
        if resultTree is None:
            raise TedTalksVideoNotFound("No TedTalks Video matches found for search value (%s)" % title)
 
        searchResults = resultTree.xpath('//result//item')
        if not len(searchResults):
            raise TedTalksVideoNotFound("No TedTalks Video matches found for search value (%s)" % title)
 
        return searchResults
        # end searchTitle()
 
 
    def searchForVideos(self, title, pagenumber):
        """Common name for a video search. Used to interface with MythTV plugin NetVision
        """
        # Get tedtalks_config.xml
        self.getUserPreferences()
 
        if self.config['debug_enabled']:
            print("self.tedtalks_config:")
            sys.stdout.write(etree.tostring(self.tedtalks_config, encoding='UTF-8', pretty_print=True))
            print()
 
        # Easier for debugging
#        print self.searchTitle(title, pagenumber, self.page_limit)
#        print
#        sys.exit()
 
        try:
            data = self.searchTitle(title, pagenumber, self.page_limit)
        except TedTalksVideoNotFound as msg:
            sys.stderr.write("%s\n" % msg)
            sys.exit(0)
        except TedTalksUrlError as msg:
            sys.stderr.write('%s\n' % msg)
            sys.exit(1)
        except TedTalksHttpError as msg:
            sys.stderr.write(self.error_messages['TedTalksHttpError'] % msg)
            sys.exit(1)
        except TedTalksRssError as msg:
            sys.stderr.write(self.error_messages['TedTalksRssError'] % msg)
            sys.exit(1)
        except Exception as e:
            sys.stderr.write("! Error: Unknown error during a Video search (%s)\nError(%s)\n" % (title, e))
            sys.exit(1)
 
        # Create RSS element tree
        rssTree = etree.XML(self.common.mnvRSS+'</rss>')
 
        # Set the paging values
        if len(data) == self.page_limit:
            self.channel['channel_returned'] = len(data)
            self.channel['channel_startindex'] = len(data)+(self.page_limit*(int(pagenumber)-1))
            self.channel['channel_numresults'] = len(data)+(self.page_limit*(int(pagenumber)-1)+1)
        else:
            self.channel['channel_returned'] = len(data)+(self.page_limit*(int(pagenumber)-1))
            self.channel['channel_startindex'] = len(data)
            self.channel['channel_numresults'] = len(data)
 
        # Add the Channel element tree
        channelTree = self.common.mnvChannelElement(self.channel)
        rssTree.append(channelTree)
 
        for item in data:
            channelTree.append(item)
 
        # Output the MNV search results
        sys.stdout.write('<?xml version="1.0" encoding="UTF-8"?>\n')
        sys.stdout.write(etree.tostring(rssTree, encoding='UTF-8', pretty_print=True))
        sys.exit(0)
    # end searchForVideos()
 
    def displayTreeView(self):
        '''Gather all videos for each TedTalks show
        Display the results and exit
        '''
        self.mashups_api.page_limit = self.page_limit
        self.mashups_api.grabber_title = self.grabber_title
        self.mashups_api.mashup_title = self.mashup_title
        self.mashups_api.channel_icon = self.channel_icon
        self.mashups_api.mashup_title = 'tedtalks'
 
        # Easier for debugging
#        self.mashups_api.displayTreeView()
#        print
#        sys.exit(1)
 
        try:
            self.mashups_api.Search = False
            self.mashups_api.displayTreeView()
        except Exception as e:
            sys.stderr.write("! Error: During a TedTalks Video treeview\nError(%s)\n" % (e))
            sys.exit(1)
 
        sys.exit(0)
    # end displayTreeView()
# end Videos() class