Ticket #4599: myth_archive.pl

#!/usr/bin/perl
$Version  = 'MythTV Recording Archiver v0.3';

# MythTV Recording Archiver
#
# Written by Mark R. Buechler 03/01/07
# License: GPL2

sub usage
  {
    die <<"EndUsage";
$Version

Usage:

Required: (at least one of)
     -archive           : Run in archive mode.
     -cleanup <mode>    : Run in cleanup mode where <mode> is a comma seperated
                          list of abandoned, zerobyte, metadata, deleted or all

Archive Arguments:
     -from <group>      : Archive recordings from storage group <group>.
     -to <group>        : Archive recordings to storage group <group>.

Archive Options:
     -ignorelivetv      : Ignore LiveTV recordings.
     -hostname <host>   : Only archive recordings which exist on
                          Myth backend <host>.
Debug:
     -debug             : Don't archive or delete anything.

Cleanup Modes:
        abandoned - Delete recordings with missing metadata.
        zerobyte  - Delete all zero-byte recordings and associated metadata.
        metadata  - Delete metadata for recordings which don't exist.
        deleted   - Delete recordins and metadata which exist in the 'Deleted'
                     storage group.
        all       - Perform all cleanup types.

Examples:
    Archive recordings from the LiveTV group to the Default group:
       myth_archive.pl -archive -from LiveTV -to Default -ignorelivetv

    Perform some cleanup of orphoned (abandoned) recordings:
       myth_archive.pl -cleanup abandoned

Notes:
       If -hostname option is not used, you must ensure that storage group
        paths are identical across all MythTV backends.

EndUsage
  }

use Getopt::Long;
use Carp qw(cluck);
use IO::File;
use DirHandle;
use DBI;

use File::Copy;
use strict;

# Pull in the MythTV module
#use lib '/opt/mns/mythtv/share/perl';
use MythTV;

my $TRUE  = 1;
my $FALSE = 0;

my $LIVETV_AUTOEXPIRE = 9999;
my $DELETED_RECGROUP = 'Deleted';

# Operational modes
my $MODE_ABANDONED = 0x01;
my $MODE_ZEROBYTE  = 0x02;
my $MODE_METADATA  = 0x04;
my $MODE_DELETED   = 0x08;
my $MODE_ALL       = 0x0f;
my $MODE_ARCHIVE   = 0x10;

my $DBH;
my $MYTH;
my $_DEBUG_;

&main();

sub getArgs {
        my $modes;
        my $from;
        my $to;
        my $hostname;
        my $ignorelivetv;
        my $cleanupmode;
        my $archivemode;
        my $batchmode;
        my $force;
        my $debug;

        my $p = new Getopt::Long::Parser;

        if (!$p->getoptions('from=s'            => \$from,
                            'to=s'              => \$to,
                            'hostname=s'        => \$hostname,
                            'ignorelivetv'      => \$ignorelivetv,
                            'archive'           => \$archivemode,
                            'cleanup=s'         => \$cleanupmode,
                            'batch'             => \$batchmode,
                            'force'             => \$force,
                            'debug'             => \$debug)) {
                print "Invalid parameters given.\n";
                usage();
        }

        $force = $TRUE if (defined($force));
        $ignorelivetv = $TRUE if (defined($ignorelivetv));
        $batchmode = $TRUE if (defined($batchmode));
        $cleanupmode =~ tr/A-Z/a-z/;
        $_DEBUG_ = $TRUE if (defined($debug));

        if ($archivemode && (!$from || !$to)) {
                print "Please specify -from and -to with -archive.\n";
                usage();
        }

        if ($cleanupmode && $hostname && !$force) {
                print "\nDANGER: This can be a BAD idea! Use -debug before trying this!\n".
                  "If you really want this, specify -force with -cleanup and -hostname.\n\n";
                exit 1;
        }

        if ($cleanupmode) {
                foreach my $mode (split(/\,/, $cleanupmode)) {
                        if ($mode eq 'abandoned') {
                                $modes |= $MODE_ABANDONED;
                        } elsif ($mode eq 'zerobyte') {
                                $modes |= $MODE_ZEROBYTE;
                        } elsif ($mode eq 'metadata') {
                                $modes |= $MODE_METADATA;
                        } elsif ($mode eq 'deleted') {
                                $modes |= $MODE_DELETED;
                        } elsif ($mode eq 'all') {
                                $modes |= $MODE_ALL;
                        } else {
                                print "Invalid mode '$mode' specified.\n";
                                usage();
                        }
                }
        }

        if (defined($archivemode)) {
                $modes |= $MODE_ARCHIVE;
        }

        if (($from eq 'LiveTV') && !$ignorelivetv && !$force) {
                print "Your source group is $from but you are not ignoring LiveTV recordings, ".
                  "please use -force option.\n";
                exit 1;
        }

        return($modes, $from, $to, $hostname, $ignorelivetv, $batchmode);
}

sub main {
        my($modes, $from, $to, $hostname, $ignorelivetv, $batchmode) = getArgs();
        my $samefs = $FALSE;
        my %recordingLookup;
        my %abandoned;

        print modesToString($modes);

        $MYTH = new MythTV();

        $DBH = $MYTH->{'dbh'};

        my $storage = getStorageGroups($hostname);
        my $recordings = getRecordings($hostname);
        my($stored, $cifs, $thumbs) = getStoredRecordings($storage);

        if ($modes & $MODE_ARCHIVE) {
                my $fromDir = $$storage{$from};
                my $toDir = $$storage{$to};

                die("FATAL: Storage group '$from' is not defined in database, aborting!\n") if (!$fromDir);
                die("FATAL: Storage group '$to' is not defined in database, aborting!\n") if (!$toDir);

                my($mpFrom, undef, undef, undef) = getMountpoint($fromDir);
                my($mpTo, undef, undef, undef) = getMountpoint($toDir);

                if ($mpFrom eq $mpTo) {
                        print "-> From and to storage groups reside on the same filesystem, ".
                          "moving recordings rather than copying/deleting.\n\n";
                        $samefs = $TRUE;
                } else {
                        print "-> From and to storage groups reside on different filesystems, ".
                          "copying/deleting recordings rather than moving.\n\n";
                }
        }

        # Some recordings may have moved, figure out which ones and update them
        foreach my $group (keys %{$recordings}) {
                foreach my $recording (keys %{$$recordings{$group}}) {
                        my $recgroup = $$recordings{$group}->{$recording}->{'RECGROUP'};
                        my $chanid = $$recordings{$group}->{$recording}->{'CHANID'};
                        my $starttime = $$recordings{$group}->{$recording}->{'START'};
                        my $title = $$recordings{$group}->{$recording}->{'TITLE'};
                        my $subtitle = $$recordings{$group}->{$recording}->{'SUBTITLE'};

                        $recordingLookup{$recording}++;

                        my $_group = findRecording($storage, $recording, $chanid, $starttime);

                        if (!defined($_group)) {
                                if ($modes & $MODE_METADATA) {
                                        print "\tDeleting meta-data for missing recording '$recording' in group '$group'.\n";

                                        deleteFromDb($chanid, $starttime, $recording, $recgroup);
                                } else {
                                        print "\tStorage group '$group': problem found with recording title '$title - ".
                                          "$subtitle'.\n";
                                }

                                next;
                        }

                        updateStorageGroup($chanid, $starttime, $recording, $group)
                          if (defined($_group) && ($group ne $_group));
                }
        }

        # Update
        $recordings = getRecordings($hostname);

        # Catch recording files still lingering around
        my %totalSize;
        foreach my $group (keys %{$stored}) {
                foreach my $recording (keys %{$$stored{$group}}) {
                        if (!$recordingLookup{$recording}) {
                                my $size = $$stored{$group}->{$recording};
                                $abandoned{$group}->{$recording} = $size;
                                print "WARNING: Found recording file '$recording' in storage group '$group'\n  ".
                                  "of size $size with no entry in database!\n";
                                $totalSize{$group} += $size;
                        }
                }
        }

        foreach my $group (keys %{$cifs}) {
                foreach my $recording (keys %{$$cifs{$group}}) {
                        my $size = $$cifs{$group}->{$recording};
                        $abandoned{$group}->{$recording} = $size;
                        print "WARNING: Found recording file '$recording' in storage group '$group'\n  ".
                          "of size $size with no entry in database!\n";
                        $totalSize{$group} += $size;
                }
        }

        foreach my $group (keys %totalSize) {
                my $size = $totalSize{$group};
                print "Found a total of $size bytes in storage group '$group' with no database entries!\n";
        }

        if ($modes & $MODE_ABANDONED) {
                if (!$batchmode) {
                        print "Preparing to cleanup recordings, press ctrl-c within 10 seconds to abort.\n";
                        sleep 10;
                }

                foreach my $group (keys %abandoned) {
                        foreach my $recording (keys %{$abandoned{$group}}) {
                                my $size = $abandoned{$group}->{$recording};
                                print "\tDeleting recording '$recording' in storage group '$group' of size $size bytes..\n";
                                unlinkFile($$storage{$group}, $recording);

                                if ($$thumbs{$recording}) {
                                        foreach my $thumb (@{$$thumbs{$recording}}) {
                                                print "\tDeleting thumbnail '$thumb'..\n";
                                                unlinkFile($$storage{$group}, $thumb);
                                        }
                                }
                        }
                }

                if ($modes & $MODE_DELETED) {
                        # Take care of recordings in the 'Deleted' group.
                        foreach my $group (keys %{$recordings}) {
                                foreach my $recording (keys %{$$recordings{$group}}) {
                                        my $recgroup = $$recordings{$group}->{$recording}->{'RECGROUP'};
                                        my $chanid = $$recordings{$group}->{$recording}->{'CHANID'};
                                        my $starttime = $$recordings{$group}->{$recording}->{'START'};

                                        if ($recgroup eq $DELETED_RECGROUP) {
                                                print "\tDeleting recording '$recording' in storage group '$group' ".
                                                  "in recgroup '$recgroup'..\n";
                                                unlinkFile($$storage{$group}, $recording);

                                                print "\tDeleting metadata for recording '$recording' in storage ".
                                                  "group '$group' in recgroup '$recgroup'..\n";
                                                deleteFromDb($chanid, $starttime, $recording, $DELETED_RECGROUP);

                                                if ($$thumbs{$recording}) {
                                                        foreach my $thumb (@{$$thumbs{$recording}}) {
                                                                print "\tDeleting thumbnail '$thumb'..\n";
                                                                unlinkFile($$storage{$group}, $thumb);
                                                        }
                                                }
                                        }
                                }
                        }
                }
        }

        foreach my $recording (keys %{$$recordings{$from}}) {
                my $size = recordingSize($$storage{$from}, $recording);
                my $recgroup = $$recordings{$from}->{$recording}->{'RECGROUP'};
                my $chanid = $$recordings{$from}->{$recording}->{'CHANID'};
                my $starttime = $$recordings{$from}->{$recording}->{'START'};
                my $title = $$recordings{$from}->{$recording}->{'TITLE'};
                my $subtitle = $$recordings{$from}->{$recording}->{'SUBTITLE'};
                my $islivetv = $$recordings{$from}->{$recording}->{'LIVETV'};

                $islivetv = 'LiveTV' if ($islivetv);

                if ($size == 0) {
                        if ($modes & $MODE_ZEROBYTE) {
                                my $recgroup = $$recordings{$from}->{$recording}->{'RECGROUP'};
                                print "\tDeleting zero-byte recording '$recording' in storage group '$from' ".
                                  "in recgroup '$recgroup'..\n";
                                unlinkFile($$storage{$from}, $recording);

                                print "\tDeleting metadata for recording '$recording' in storage group '$from' ".
                                  "in recgroup '$recgroup'..\n";
                                deleteFromDb($chanid, $starttime, $recording, $recgroup);

                                if ($$thumbs{$recording}) {
                                        foreach my $thumb (@{$$thumbs{$recording}}) {
                                                print "\tDeleting thumbnail '$thumb'..\n";
                                                unlinkFile($$storage{$from}, $thumb);
                                        }
                                }
                        } else {
                                print "WARNING: File '$recording' for $islivetv recording of '$title - $subtitle' ".
                                  "is 0 bytes in size!\n";
                        }
                } elsif ($modes & $MODE_ARCHIVE) {
                        if (!$ignorelivetv || ($ignorelivetv && !$islivetv)) {
                                my $fromDir = $$storage{$from};
                                my $toDir = $$storage{$to};

                                print "\tArchiving recording '$recording':\n\t\tTitle '$title - $subtitle' from ".
                                  "'$fromDir' to '$toDir'..\n";

                                if (!archiveFile($fromDir, $toDir, $recording, $samefs)) {
                                        updateStorageGroup($chanid, $starttime, $recording, $to);
                                }

                                if ($$thumbs{$recording}) {
                                        foreach my $thumb (@{$$thumbs{$recording}}) {
                                                archiveFile($fromDir, $toDir, $thumb, $samefs);
                                        }
                                }
                        }
                }
        }

        # Now print some stats..
        print "\n\tStorage Group\tSize\tPortion\n";
        print "\t----------------------------------------\n";

        my $total = 0;
        foreach my $group (keys %{$recordings}) {
                next if (!$group);
                my $group_s = 0;
                my(undef, $mp_size, undef, undef) = getMountpoint($$storage{$group});
                $mp_size = $mp_size * 1024;
                $mp_size -= (.05 * $mp_size);

                foreach my $recording (keys %{$$recordings{$group}}) {
                        my $size = recordingSize($$storage{$group}, $recording);
                        $group_s += $size;
                }

                my $portion = round((($group_s / $mp_size) * 100), 2) if ($mp_size);

                print "\t$group\t\t$group_s\t$portion%\n";
                $total += $group_s;
        }

        print "\t----------------------------------------\n";
        print "\tTotal:\t\t$total\n\n";

        print "All done!\n";
}

sub getStorageGroups {
        my $hostname = shift;
        my %groups;
        my $where;

        $where = "WHERE hostname = '$hostname'" if ($hostname);

        my $sth = $DBH->prepare("SELECT groupname, dirname FROM storagegroup $where");
        $sth->execute();

        while (my $row = $sth->fetchrow_hashref()) {
                my $path = $row->{'dirname'};
                $path =~ s/\/+$//;

                $groups{$row->{'groupname'}} = $path;
        }

        $sth->finish();

        return \%groups;
}

sub getRecordings {
        my $hostname = shift;
        my %recordings;
        my $where;

        $where = "WHERE hostname = '$hostname'" if ($hostname);

        my $sth = $DBH->prepare("SELECT chanid, starttime, basename, title, subtitle, ".
          "autoexpire, storagegroup, recgroup FROM recorded $where");
        $sth->execute();

        while (my $row = $sth->fetchrow_hashref()) {
                $recordings{$row->{'storagegroup'}}->{$row->{'basename'}}->{'TITLE'} =
                  $row->{'title'};
                $recordings{$row->{'storagegroup'}}->{$row->{'basename'}}->{'SUBTITLE'} =
                  $row->{'subtitle'};
                $recordings{$row->{'storagegroup'}}->{$row->{'basename'}}->{'CHANID'} =
                  $row->{'chanid'};
                $recordings{$row->{'storagegroup'}}->{$row->{'basename'}}->{'START'} =
                  $row->{'starttime'};
                $recordings{$row->{'storagegroup'}}->{$row->{'basename'}}->{'RECGROUP'} =
                  $row->{'recgroup'};
                $recordings{$row->{'storagegroup'}}->{$row->{'basename'}}->{'LIVETV'} =
                  ($row->{'autoexpire'} >= $LIVETV_AUTOEXPIRE) ? $TRUE : $FALSE;
        }

        $sth->finish();

        return \%recordings;
}

sub getStoredRecordings {
        my $storage = shift;
        my %stored;
        my %cifs;
        my %thumbs;

        foreach my $group (keys %{$storage}) {
                my $path = $$storage{$group};
                my $dir = new DirHandle($path);

                die("FATAL: Directory '$path' for storage group '$group' does not exist!\n")
                  if (!defined($dir));

                while (defined(my $entry = $dir->read())) {
                        next if (-d $entry);
                        next if (($entry !~ /\.mpg$/i) &&
                                 ($entry !~ /\.nuv$/i) &&
                                 ($entry !~ /\.png$/i) &&
                                 ($entry !~ /^cifs/));

                        if ($entry =~ /\.mpg$/) {
                                $stored{$group}->{$entry} = recordingSize($path, $entry);
                        } elsif ($entry =~ /\.png$/) {
                                if ($entry =~ /^(.+\.(mpg|nuv))\./i) {
                                        push @{$thumbs{$1}}, $entry;
                                }
                        } elsif ($entry =~ /^cifs/) {
                                $cifs{$group}->{$entry} = recordingSize($path, $entry);
                        }
                }
        }

        return(\%stored, \%cifs, \%thumbs);
}

sub updateStorageGroup {
        my $chanid = shift;
        my $starttime = shift;
        my $recording = shift;
        my $group = shift;

        my $sql = "UPDATE recorded SET storagegroup = '$group' ".
          "WHERE chanid = $chanid AND starttime = '$starttime' AND basename = '$recording'";

        if ($_DEBUG_) {
                print "DBG: updateStorageGroup(): $sql\n";
        } else {
                my $sth = $DBH->prepare($sql);
                $sth->execute();
                $sth->finish();
        }
}

sub findRecording {
        my $storage = shift;
        my $recording = shift;
        my $chanid = shift;
        my $starttime = shift;
        my $found;

        foreach my $group (keys %{$storage}) {
                my $directory = $$storage{$group};

                if (-e "$directory/$recording") {
                        if ($found) {
                                print "WARNING: Found recording '$recording' in multiple groups, ignoring!\n";
                                return undef;
                        }

                        $found = $group;
                }
        }

        if (!$found) {
                print "WARNING: Unable to find recording '$recording' in any storage group!\n";
                return undef;
        }

        return $found;
}

sub deleteFromDb {
        my $chanid = shift;
        my $starttime = shift;
        my $basename = shift;
        my $recgroup = shift;

        doSql("DELETE FROM recorded WHERE chanid = $chanid AND starttime = '$starttime' AND ".
          "basename = '$basename' AND recgroup = '$recgroup'");
        doSql("DELETE FROM recordedcredits WHERE chanid = $chanid AND starttime = '$starttime'");
        doSql("DELETE FROM recordedmarkup WHERE chanid = $chanid AND starttime = '$starttime'");
        doSql("DELETE FROM recordedprogram WHERE chanid = $chanid AND starttime = '$starttime'");
        doSql("DELETE FROM recordedrating WHERE chanid = $chanid AND starttime = '$starttime'");
        doSql("DELETE FROM recordedseek WHERE chanid = $chanid AND starttime = '$starttime'");
}

sub archiveFile {
        my $fromDir = shift;
        my $toDir = shift;
        my $file = shift;
        my $samefs = shift;

        if (!-e "$fromDir/$file") {
                print "WARNING: Attempting to archive the file '$fromDir/$file' which doesn't exist!\n";
                return $TRUE;
        }

        if (-e "$toDir/$file") {
                print "WARNING: Recording '$file' already exists in destination group, ignoring!\n";
                return $TRUE;
        }

        if (!$samefs && !checkForSpace($fromDir, $toDir, $file)) {
                print "Storage group directory '$toDir' is too full to achive recording '$file', ignoring.\n";
                return $TRUE;
        }

        if ($_DEBUG_) {
                print "DBG: Would archive '$fromDir/$file' to '$toDir/$file' mode ".($samefs ? "mv" : "cp")."\n";
        } else {
                if ($samefs) {
                        if (!move("$fromDir/$file", "$toDir/$file")) {
                                print "WARNING: Failed to archive/move recording $file from $fromDir to $toDir: $!\n";
                                return $TRUE;
                        }
                } else {
                        if (!copy("$fromDir/$file", "$toDir/$file")) {
                                print "WARNING: Failed to archive/copy recording $file from $fromDir to $toDir: $!\n";
                                return $TRUE;
                        }
                }       
        }

        my $orig_size = recordingSize($fromDir, $file);
        my $copy_size = recordingSize($toDir, $file);

        if (!$samefs && ($orig_size != $copy_size)) {
                print "WARNING: Failed to copy recording '$file' from $fromDir to $toDir, file sizes don't match!\n"
                  if (!$_DEBUG_);
                return $TRUE;
        } elsif (!$samefs) {
                unlinkFile($fromDir, $file);
        }

        return $FALSE;
}

sub unlinkFile {
        my $fromDir = shift;
        my $file = shift;

        return $TRUE if (!$fromDir || !$file);

        if (-e "$fromDir/$file") {
                if ($_DEBUG_) {
                        print "DBG: Would unlink \"$fromDir/$file\"\n";
                } else {
                        if (!unlink "$fromDir/$file") {
                                print "WARNING: Failed to unlink file $file in $fromDir: $!\n";
                                return $TRUE;
                        }
                }
        }

        return $FALSE;
}

sub isFileInUse {
        my $file = shift;

        my $result = `fuser $file`;

        $result =~ s/\s+$//;

        my(undef, $pids) = split(/\:/, $result);

        return $pids;
}

sub checkForSpace {
        my $fromDir = shift;
        my $toDir = shift;
        my $file = shift;
        my $mount;
        my $available;
        my $percentage;

        my(undef, undef, $available, $percentage) = getMountpoint($toDir);

        my $file_size = recordingSize($fromDir, $file);

        return $FALSE if (($available < $file_size) || ($percentage > 98));
        return $TRUE;
}

sub getMountpoint {
        my $directory = shift;
        my $mp;
        my $size;
        my $free;
        my $percentage;

        my $buff = `df -kP $directory`;

        foreach my $line (split(/\n/, $buff)) {
                ($mp, $size, undef, $free, $percentage) = split(/\s+/, $line, 5)
                  if ($line !~ /^Filesystem/);
        }

        $free = $free * 1024;
        $percentage =~ s/\%//;

        return($mp, $size, $free, $percentage);
}

sub recordingSize {
        my $directory = shift;
        my $recording = shift;

        if (!-e "$directory/$recording") {
                return -1;
        }

        my $size = -s "$directory/$recording";

        return $size;
}

sub round {
        my $num = shift;
        my $place = shift;

        my($whole, $decimal) = split(/\./, $num);
        my $decimal_t = substr($decimal, 0, $place);

        if ($decimal_t) {
                my $rounder = substr($decimal, $place, 1);
                $decimal_t += 1 if ($rounder && $rounder > 5);
        } else {
                $decimal_t = 0;
        }

        return $whole.".".$decimal_t;
}

sub modesToString {
        my $modes = shift;

        if (!$modes) {
                return "\nNo mode specified, checking only.\n\n";
        }

        my $string = "\nWorking in the following modes:\n";

        if ($modes & $MODE_ARCHIVE) {
                $string .= "\t-> Archive Recordings\n";
        }

        if ($modes & $MODE_ABANDONED) {
                $string .= "\t-> Delete abandoned recordings\n";
        }

        if ($modes & $MODE_ZEROBYTE) {
                $string .= "\t-> Delete zero-byte recordings\n";
        }

        if ($modes & $MODE_METADATA) {
                $string .= "\t-> Delete meta-data for missing recordings\n";
        }

        if ($modes & $MODE_DELETED) {
                $string .= "\t-> Delete recordings in 'Deleted' group\n";
        }

        $string .= "\n";

        return $string;
}

sub doSql {
        my $sql = shift;

        if ($_DEBUG_) {
                print "DBG: $sql\n";
        } else {
                my $sth = $DBH->prepare($sql);

                $sth->execute();
                $sth->finish();
        }
}