Ticket #2734: metadata_script_improvements.patch
File metadata_script_improvements.patch, 17.6 KB (added by , 17 years ago) |
---|
-
mythvideo/mythvideo/scripts/find_meta.py
old new 5 5 directory with multiple video files. Perform movie data lookups using the 6 6 imdbpy.py script. 7 7 8 Written by Pekka JÀÀskelÀinen (gmail: pekka.jaaskelainen) 2006 .8 Written by Pekka JÀÀskelÀinen (gmail: pekka.jaaskelainen) 2006-2007. 9 9 10 10 The metadata is searched with following steps: 11 11 … … 29 29 Includes an interactive mode which makes the script ask user for input in case there are 30 30 titles for which IMDb queries cannot be done unambiguously. 31 31 32 A crontab entry for the script to insert automatically data of new video files to 33 MythDB every hour: 34 35 0 * * * * find_meta.py -fdr /videos 36 37 In which the /videos is the root of your MythVideo files. 38 39 TODO: 40 - fetch and store categories 41 - fetch video extension list from MythDB, if available 42 - detect if metadata already exist before starting to scan the file: 43 needs to detect both the MythDB insertion and .metadata files 44 - if a .metadata is found, just insert it into the MythDB without fetching 45 the info again 46 - insert the poster path to the MythDB 47 32 48 """ 33 49 34 50 import sys … … 38 54 import glob 39 55 import fileinput 40 56 import imdbpy 57 import shlex 58 41 59 try: 42 60 # If loaded, get neater input prompt for interactive questions. 43 61 import readline 44 62 except: 45 63 pass 46 64 65 try: 66 # If found, we can insert data directly to MythDB 67 import MySQLdb 68 db_support = True 69 except: 70 db_support = False 71 47 72 from stat import * 48 73 49 74 verbose=False 50 overwrite=False 75 51 76 interactive=False 52 77 recursive=False 78 dbimport=False 79 80 # Import metadata from .metadata files if found. 81 import_from_files=False 82 83 # Overwrite the old metadata if found. 84 overwrite=False 53 85 54 86 videoExtensions = ["avi", "mpg", "wmv"] 55 87 … … 60 92 61 93 # Directories to skip in the recursive scan (matched to the rightmost 62 94 # part of the directory). 63 skipDirs = ["/Sample", "/Sub" ]95 skipDirs = ["/Sample", "/Sub", "/VIDEO_TS"] 64 96 65 97 def print_verbose(string): 66 98 global verbose … … 68 100 print string 69 101 return 70 102 103 db = None 104 def init_db(): 105 global db 106 try: 107 config = shlex.shlex(open(os.path.expanduser('~/.mythtv/mysql.txt'))) 108 except: 109 print "Error opening ~/.mythtv/mysql.txt" 110 return False 111 112 113 token = config.get_token() 114 db_host = db_user = db_password = None 115 while token != config.eof and (db_host == None or db_user == None or db_password == None): 116 if token == "DBHostName": 117 if config.get_token() == "=": 118 db_host = config.get_token() 119 elif token == "DBUserName": 120 if config.get_token() == "=": 121 db_user = config.get_token() 122 elif token == "DBPassword": 123 if config.get_token() == "=": 124 db_password = config.get_token() 125 126 token = config.get_token() 127 db = MySQLdb.connect(user=db_user, host=db_host, passwd=db_password, 128 db="mythconverg") 129 print_verbose("Database connection successful.") 130 return True 131 71 132 def find_imdb_id_from_text_file(textFile): 72 133 73 134 if os.access(textFile, os.R_OK): … … 112 173 title = title[0:lowest_cutpoint] 113 174 return title.strip() 114 175 115 def save_metadata(fileName, metadata): 176 def save_metadata_to_mythdb(videopath, metadata): 177 global overwrite, db 178 179 # Drop the trailing '/' from the path 180 if videopath.endswith('/'): 181 videopath = videopath[0:-1] 182 videopath = os.path.abspath(videopath) 183 184 print_verbose("Inserting metadata to MythDB for %s." % videopath) 185 # Look for existing metadata entry for the video 186 c = db.cursor() 187 c.execute(""" 188 SELECT intid, title, director, plot, rating, inetref, year, 189 userrating, length, filename, coverfile, childid, playcommand 190 FROM videometadata 191 WHERE filename = %s""", (videopath,)) 192 row = c.fetchone() 193 c.close() 194 195 (intid, title, director, plot, rating, inetref, year, 196 userrating, length, filename, coverfile, childid, playcommand) = \ 197 (None, None, None, None, None, None, 198 0.0, None, 0, None, None, -1, "") 199 200 if row is not None: 201 # There was an entry already, update the old values, but 202 # do not overwrite old values by default. 203 if not overwrite: 204 (intid, title, director, plot, rating, inetref, year, 205 userrating, length, filename, coverfile, childid, playcommand) = row 206 print_verbose("Metadata already exist in MythDB, augmenting it.") 207 else: 208 intid = row[0] 209 210 else: 211 print_verbose("No metadata in MythDB, creating a new one.") 212 # Create a new empty entry at this point so we can use the common UPDATE code 213 # to actually insert the data. 214 c = db.cursor() 215 c.execute("""INSERT INTO videometadata(filename) VALUES(%s)""", (videopath,)) 216 intid = c.lastrowid 217 c.close() 218 219 def parse_metadata(variable, oldvalue, emptyvalue="", meta=metadata): 220 global overwrite 221 if not overwrite and oldvalue is not None and oldvalue != emptyvalue: 222 return oldvalue 223 for line in meta: 224 beginning = variable + ":" 225 if line.startswith(beginning): 226 return line[len(beginning):].strip() 227 return None 228 229 title = parse_metadata('Title', title) 230 year = parse_metadata('Year', year, 0) 231 232 if title is None or year is None: 233 return 234 235 236 director = parse_metadata('Director', director) 237 238 if director == None: 239 director = "Unknown" 240 241 plot = parse_metadata('Plot', plot, "None") 242 userrating = parse_metadata('UserRating', userrating, 0.0) 243 244 try: 245 float(userrating) 246 except: 247 userrating = 0.0 248 249 rating = parse_metadata('MovieRating', rating, "Unknown") 250 251 if rating is None: 252 rating = "Unknown" 253 254 length = parse_metadata('Runtime', length, 0) 255 256 try: 257 if int(length) == 0: 258 length = 0 259 except: 260 length = 0 261 262 inetref = parse_metadata('IMDb', inetref, '00000000') 263 264 if inetref == None: 265 inetref = '00000000' 266 267 filename = videopath 268 269 if coverfile is None: 270 coverfile = "No cover" 271 272 #print "id: ", intid, "\n", title, director, plot, rating, inetref, year, userrating, length, filename, coverfile 273 c = db.cursor() 274 c.execute(""" 275 UPDATE videometadata 276 SET showlevel = 1, browse = 1, childid = %s, playcommand = %s, title = %s, 277 director = %s, plot = %s, rating = %s, inetref = %s, 278 year = %s, userrating = %s, length = %s, filename = %s, coverfile = %s 279 WHERE intid = %s""", 280 (childid, playcommand, title, director, plot, rating, inetref, year, userrating, length, 281 filename, coverfile, intid)) 282 c.close() 283 284 285 286 287 def save_metadata_to_file(fileName, metadata): 288 global overwrite 116 289 117 290 if os.path.exists(fileName) and not overwrite: 118 291 print_verbose("Metadata already exists, not overwriting.") 292 return 119 293 120 294 if metadata is not None: 121 295 print_verbose("Writing metadata to '%s'" % fileName) 122 296 f = open(fileName, 'w') 123 f.write(metadata.encode("utf8")) 297 for line in metadata: 298 f.write(line.encode("utf8")) 124 299 f.close() 125 300 print_verbose("Metadata:") 126 301 print_verbose(metadata) 127 302 303 def save_metadata(videopath, metadata_filename, metadata): 304 global dbimport 305 save_metadata_to_file(metadata_filename, metadata) 306 if dbimport: 307 save_metadata_to_mythdb(videopath, metadata) 128 308 129 309 def find_metadata_for_video_path(pathName): 310 """ 311 Returns a pair [source_nfo, metadata] in which the first denotes the possible 312 .nfo file from which IMDb ID was found, and latter a list of metadata found using 313 the IMDb ID. 314 """ 130 315 131 316 global interactive 132 317 … … 146 331 dirName + "/imdb.url", 147 332 dirName + "/" + file_body + ".nfo", 148 333 dirName + "/" + file_body + ".imdb"] 149 150 334 151 152 # Add rest of the .nfos to the list of the scanned ones. 335 # Add rest of the .nfos to the end of the list. 153 336 for nfo in glob.glob(dirName + "/*.nfo"): 154 337 if nfo not in nfos: 155 338 nfos.append(nfo) … … 177 360 178 361 index = 0 179 362 if len(candidates) > 1: 363 364 print "Got multiple candidates for the title search '%s'. " % title 365 for candidate in candidates: 366 print "%s) %s (%d)" % (candidate[0], candidate[1], candidate[2]) 367 180 368 if interactive: 181 print "Got multiple candidates for the title search '%s'. " % title182 print "Which one is the correct title?"183 for i in range(len(candidates)):184 print "%d) %s (%d)" % (i, candidates[i][1], candidates[i][2])185 print "N) None of the above"186 369 answer = raw_input("?)") 187 370 if answer is None or len(answer) == 0: 188 371 return [None, None] 189 372 190 if answer.lower() == "n": 191 return [None, None] 192 else: 193 try: 194 ans = int(answer) 195 if 0 <= ans < len(candidates): 196 index = ans 197 else: 198 return [None, None] 199 except: 200 print_verbose("Illegal index.") 201 return [None, None] 202 print_verbose("Chose %d" % index) 373 print_verbose("Chose %s" % answer) 374 imdb_id = answer 203 375 else: 204 print "Multiple candidates found for the movie and not in interactive mode." 205 return [None, None] 206 207 imdb_id = candidates[index][0] 376 return [None, None] 208 377 else: 209 378 print "Couldn't find IMDb ID for '%s'" % pathName 210 379 return [None, None] … … 223 392 224 393 return "Files:%s" % (",".join(videoPaths)) + "\n" 225 394 395 def load_metadata_file(metadata_filename): 396 """ 397 Loads a metadata file if found, returns None otherwise. 398 """ 399 metadata = None 400 try: 401 f = open(metadata_filename) 402 metadata = f.readlines() 403 f.close() 404 except: 405 pass 406 return metadata 407 226 408 def detect_dvd_backup(dirName): 227 409 """ 228 410 If the given directory is detected as a directory with a dvd backup, meta data is 229 411 searched for the directory title name (in addition to the directory-wide imdb files). 230 412 """ 413 global import_from_files 231 414 videoTs = dirName + "/VIDEO_TS" 232 415 if not (os.path.exists(videoTs) and os.path.isdir(videoTs)): 233 416 return False … … 236 419 237 420 metadata_target = dirName + "/" + dirMetadataFileName 238 421 239 if os.path.exists(metadata_target) and not overwrite: 240 print_verbose("Metadata already exists, not overwriting.") 241 return True 242 243 [src_nfo, metadata] = find_metadata_for_video_path(dirName + "/VIDEO_TS") 422 metadata = None 423 if import_from_files: 424 metadata = load_metadata_file(metadata_target) 425 426 if metadata is None: 427 [src_nfo, metadata] = find_metadata_for_video_path(dirName + "/VIDEO_TS") 428 244 429 if metadata is not None: 245 save_metadata( metadata_target, metadata)430 save_metadata(dirName, metadata_target, metadata) 246 431 return True 247 432 248 433 … … 250 435 def detect_compressed_dvd_backup_dir(dirName): 251 436 """ 252 437 If the given directory is detected as a directory with one or more files of 253 a single title, fetche dthe meta data and returns true.438 a single title, fetches the meta data and returns true. 254 439 """ 255 global videoExtensions 440 global videoExtensions, import_from_files 256 441 maxFilesPerTitle = 3 257 442 258 443 foundVideos = None … … 284 469 metadata_target = dirName + "/" + dirMetadataFileName 285 470 if foundVideos is not None: 286 471 print_verbose('Compressed DVD backup directory (a.k.a. DVD rip) detected.') 287 288 if os.path.exists(metadata_target) and not overwrite: 289 print_verbose("Metadata already exists, not overwriting.") 290 return True 291 472 292 473 # Scan for the IMDb ID as usual, but store the 293 474 # metadata to video.metadata file instead, i.e., do not create .metadata for 294 475 # all videos in the directory. 295 [src_nfo, metadata] = find_metadata_for_video_path(foundVideos[0]) 476 metadata = None 477 if import_from_files: 478 metadata = load_metadata_file(metadata_target) 479 480 if metadata is None: 481 [src_nfo, metadata] = find_metadata_for_video_path(foundVideos[0]) 296 482 if metadata is not None: 297 483 # Add the Files: metadata which lists the videos in correct playing order 298 484 metadata += video_file_list_metadata(foundVideos) 299 save_metadata( metadata_target, metadata)485 save_metadata(dirName, metadata_target, metadata) 300 486 return True 301 487 302 488 return False 303 489 304 def scan_file(pathName): 305 metadata_target = strip_extension(pathName) + ".metadata"; 306 if os.path.exists(metadata_target) and not overwrite: 307 print_verbose("Metadata already exists, not overwriting.") 308 else: 490 def scan_file(pathName, imdb_id = None): 491 global import_from_files 492 493 metadata_target = strip_extension(pathName) + ".metadata"; 494 metadata = None 495 if import_from_files: 496 metadata = load_metadata_file(metadata_target) 497 498 if imdb_id is not None: 499 metadata = imdbpy.metadata_search(imdb_id) 500 501 if metadata is None: 309 502 [src_nfo, metadata] = find_metadata_for_video_path(pathName) 310 save_metadata(metadata_target, metadata) 503 504 if metadata is not None: 505 save_metadata(pathName, metadata_target, metadata) 311 506 312 507 313 def scan_directory(dirName ):508 def scan_directory(dirName, imdb_id = None): 314 509 global videoExtensions 315 510 print_verbose("Scanning directory %s..." % dirName) 316 if detect_compressed_dvd_backup_dir(dirName): 317 return 511 # Disabled until we have some kind of nice way to play dvd rips in MythVideo 512 #if detect_compressed_dvd_backup_dir(dirName): 513 # return 514 515 if imdb_id is not None: 516 metadata = imdbpy.metadata_search(imdb_id) 517 if metadata is not None: 518 save_metadata(dirName, dirName + "/video.metadata", metadata) 519 return 520 318 521 319 522 if detect_dvd_backup(dirName): 320 523 return … … 334 537 for video in foundVideos: 335 538 scan_file(video) 336 539 337 def scan(pathName): 540 def should_be_skipped(directory): 541 """ 542 Returns true in case the given directory should be skipped in 543 the scan. 544 """ 545 global skipDirs 546 if directory.endswith('/'): 547 directory = directory[0:-1] 548 549 for skip in skipDirs: 550 if directory.endswith(skip): 551 return True 552 return False 553 554 def scan(pathName, imdb_id = None): 338 555 global recursive 339 556 metadata = None 340 557 metadata_target = None … … 343 560 for root, dirs, files in os.walk(pathName): 344 561 if should_be_skipped(root): continue 345 562 scan_directory(root) 563 for f in dirs: 564 path = pathName + f 565 if os.path.islink(path): 566 scan(path) 346 567 else: 347 scan_directory(pathName )568 scan_directory(pathName, imdb_id) 348 569 elif os.path.isfile(pathName): 349 scan_file(pathName )570 scan_file(pathName, imdb_id) 350 571 else: 351 572 raise IOError("File not found") 352 573 return 353 574 354 575 def main(): 355 global verbose,overwrite,interactive,recursive 576 global verbose,overwrite,interactive,recursive,dbimport,import_from_files 356 577 357 578 p = optparse.OptionParser() 358 579 p.add_option('--version', '-v', action="store_true", default=False, 359 580 help="display 1-line describing name, version, author etc") 360 581 p.add_option('--overwrite', '-o', action="store_true", default=False, 361 help="overwrite existing metadata file(s)")582 help="overwrite existing metadata") 362 583 p.add_option('--wordy', '-w', action="store_true", default=False, 363 584 help="verbose mode, be wordy while scanning for the info") 364 585 p.add_option('--interactive', '-i', action="store_true", default=False, 365 586 help="allow the script to ask questions from the user to find the meta data") 366 587 p.add_option('--recursive', '-r', action="store_true", default=False, 367 588 help="traverse sub directories of the given directory recursively") 589 590 p.add_option('--dbimport', '-d', action="store_true", default=False, 591 help="import metadata directly to MythDB (requires MySQLdb support in Python)") 592 p.add_option('--fromfiles', '-f', action="store_true", default=False, 593 help="import data to MythDB from .metadata files if found. Requires -d.") 594 595 p.add_option('--answer', '-a', action="store", type="string", dest="imdb_id", 596 help="fetch metadata with the given IMDb ID.") 368 597 369 598 370 599 options, arguments = p.parse_args() 371 600 372 601 if options.version: 373 print "MythVideo Metadata Finder (c) Pekka J ääskeläinen 2006-2007"602 print "MythVideo Metadata Finder (c) Pekka JÀÀskelÀinen 2006-2007" 374 603 sys.exit(0) 375 604 376 605 verbose = options.wordy 377 606 overwrite = options.overwrite 378 607 interactive = options.interactive 379 608 recursive = options.recursive 609 dbimport = options.dbimport 610 import_from_files = options.fromfiles and dbimport 380 611 381 612 if len(arguments) < 1: 382 613 print "Please give the file/directory to be scanned as argument." 383 614 sys.exit(1) 615 616 if dbimport: 617 if not db_support: 618 print "You must install MySQLdb module to make direct DB importing to work" 619 sys.exit(1) 620 if not init_db(): 621 sys.exit(1) 384 622 385 623 pathName = " ".join(arguments) 386 624 … … 388 626 print "Given file does not exist." 389 627 sys.exit(1) 390 628 391 scan(pathName) 629 if options.imdb_id is not None: 630 if options.imdb_id and recursive: 631 print "Manual IMDb ID cannot be given with recursive mode." 632 sys.exit(1) 633 print_verbose("IMDb ID %s given manually." % options.imdb_id) 634 scan(pathName, options.imdb_id) 392 635 393 636 sys.exit(0) 394 637 -
mythvideo/mythvideo/scripts/imdbpy.py
old new 226 226 options, arguments = p.parse_args() 227 227 228 228 if options.version: 229 print "MythVideo IMDbPy wrapper v1.0 (c) Pekka JÀÀskelÀinen 2006"229 print "MythVideo IMDbPy wrapper (c) Pekka JÀÀskelÀinen 2006-2007" 230 230 sys.exit(0) 231 231 232 232 if options.info: 233 233 print "Uses the IMDbPy package to fetch the data, thus externalizes the actual "\ 234 234 "parsing of IMDb data to another project, hopefully reducing the maintenance burden "\ 235 "in the future , in addition supports fetching data for TV-series episodes."235 "in the future. In addition supports fetching data for TV-series episodes." 236 236 sys.exit(0) 237 237 238 238 if options.movie_search is not None: … … 242 242 elif options.poster_search is not None: 243 243 poster_search(options.poster_search) 244 244 elif options.metadata_search is not None: 245 print metadata_search(options.metadata_search) 245 print metadata_search(options.metadata_search).encode("utf8") 246 246 else: 247 247 p.print_help() 248 248 sys.exit(0)