Changeset 5 for shepherd

Timestamp:

10/04/06 00:23:03 (7 years ago)

Author:

lincoln

Message:

postprocessor support

Files:

• shepherd (modified) (34 diffs)

shepherd

@@ -3 +3 @@
 # "Shepherd"
 
-my $version = '0.2.5ng';
+my $version = '0.2.6';
 
 # A wrapper for various Aussie TV guide data grabbers
@@ -19 +19 @@
 # 0.2.2   : --check
 # 0.2.3   : Bugfix: archives correctly
-# 0.2.5ng : multi-grabber (potentially with partial data) & postprocessor support
+# 0.2.5   : multi-grabber (potentially with partial data)
+# 0.2.6   : postprocessor support
 #
 # ToDo:
@@ -28 +29 @@
 #   --desc or --ready
 # * --ready  option that says whether config is required?
-# * "Last Data" column in status
 
 use strict;
@@ -39 +39 @@
 use XMLTV::Ask;
 use DateTime::Format::Strptime;
+use POSIX qw(strftime);
 
 # ---------------------------------------------------------------------------
@@ -58 +59 @@
 
 my $GRABBER_DIR = "$CWD/grabbers";
+my $POSTPROCESSOR_DIR = "$CWD/postprocessors";
 my $ARCHIVE_DIR = "$CWD/archive";
-my $timeslot_size = (15 * 60); # 15 minute slots
-my $channel_ok_threshold_percent = 90;
+
+my $timeslot_size = (15 * 60);                  # 15 minute slots
+my $channel_ok_threshold_percent = 90;          # 90% these may need to be tweaked but look ok for now
+my $postprocessor_ok_threshold_percent = 80;    # 80% these may need to be tweaked but look ok for now
+my $postprocessor_disable_failure_threshold = 5; # number of times a postprocessor has to fail in a row before it is automatically disabled
 
 my $opt;
+my $pref_order;
+my $made_changes = 0;
 my $debug = 1;
 my $grabbers = { };
+my $postprocessors = { };
 my $preferred; # obsolete but may still exist in shepherd.conf
 my $region;
@@ -75 +83 @@
 my $langs = [ 'en' ];
 my $num_timeslots;
-my $grabber_data = { };
+my $plugin_data = { };
 my $channel_data = { };
 my $starttime, my $endtime;
+my $input_postprocess_files = "";
+my $insufficient_grabber_data = 0;
 
 # ---------------------------------------------------------------------------
@@ -123 +133 @@
 }
 
-&set_order($opt->{setorder}) if ($opt->{setorder});
-
-if ($opt->{check})
-{
-    check();
-}
+&set_order(0,$opt->{setorder}) if ($opt->{setorder});
+&check() if ($opt->{check});
 
 if ($opt->{enable} or $opt->{disable} or $opt->{setorder} or $opt->{check})
 {
-    write_config_file();
+    set_order(1) if $made_changes;
+    write_config_file() if $made_changes;
     status();
     exit;
@@ -160 +167 @@
     calc_date_range();
     grab_data();
+    postprocess_data();
+    output_data();
 }
 
@@ -165 +174 @@
 
 status();
+write_config_file();
 
 # ---------------------------------------------------------------------------
@@ -184 +194 @@
         $used_grabbers++;
 
-        $grabber_data->{$grabber}->{last_grabbed} = time;
+        $grabbers->{$grabber}->{lastdata} = time;
+        $grabbers->{$grabber}->{laststatus} = "unknown";
 
         printf "SHEPHERD: Using grabber: (%d) %s\n",$grabbers->{$grabber}->{order},$grabber;
 
         my $output = "$GRABBER_DIR/$grabber/output.xmltv";
+        $input_postprocess_files .= "$output ";
+
         my $comm = "$GRABBER_DIR/$grabber/$grabber " .
                    "--region $region " .
@@ -208 +221 @@
 
         # soak up the data we just collected
-        &soak_up_grabber_data($grabber, $output);
+        &soak_up_data($grabber, $output);
+        $grabbers->{$grabber}->{laststatus} = $plugin_data->{$grabber}->{laststatus};
 
         # check to see if we have all the data we want
-        $need_more_data = &analyze_grabber_data();
+        $need_more_data = &analyze_plugin_data($channel_ok_threshold_percent);
 
         last if (!$need_more_data);
@@ -226 +240 @@
     {
         print "SHEPHERD: Ran through ALL grabbers but still missing data!!! :(\n";
+        $insufficient_grabber_data = 1;
         return;
     }
@@ -231 +246 @@
 
 
-# interpret xmltv data from this grabber
-sub soak_up_grabber_data
+# interpret xmltv data from this grabber/postprocessor
+sub soak_up_data
 {
     my ($grabber, $output) = @_;
-    eval { $grabber_data->{$grabber}->{xmltv} = XMLTV::parsefiles($output); };
-
-    if (defined $grabber_data->{$grabber}->{xmltv}) {
-        $grabber_data->{$grabber}->{valid} = 1;
-
-        my $xmltv = $grabber_data->{$grabber}->{xmltv};
+    eval { $plugin_data->{$grabber}->{xmltv} = XMLTV::parsefiles($output); };
+
+    if (defined $plugin_data->{$grabber}->{xmltv}) {
+        $plugin_data->{$grabber}->{valid} = 1;
+
+        my $xmltv = $plugin_data->{$grabber}->{xmltv};
         my ($encoding, $credits, $chan, $progs) = @$xmltv;
-        $grabber_data->{$grabber}->{total_duration} = 0;
-        $grabber_data->{$grabber}->{programmes} = 0;
+        $plugin_data->{$grabber}->{total_duration} = 0;
+        $plugin_data->{$grabber}->{programmes} = 0;
 
         my $strptime = new DateTime::Format::Strptime( pattern => "%Y%m%d%H%M %z");
@@ -261 +276 @@
 
                 # store grabber-specific stats
-                $grabber_data->{$grabber}->{programmes}++;
-                $grabber_data->{$grabber}->{total_duration} += ($t2->epoch - $t1->epoch);
+                $plugin_data->{$grabber}->{programmes}++;
+                $plugin_data->{$grabber}->{total_duration} += ($t2->epoch - $t1->epoch);
                 $seen_progs_on_this_channel++;
-                $grabber_data->{$grabber}->{earliest_data_seen} = $t1->epoch if (!defined $grabber_data->{$grabber}->{earliest_data_seen});
-                $grabber_data->{$grabber}->{earliest_data_seen} = $t1->epoch if ($t1->epoch < $grabber_data->{$grabber}->{earliest_data_seen});
-                $grabber_data->{$grabber}->{latest_data_seen} = $t2->epoch if (!defined $grabber_data->{$grabber}->{latest_data_seen});
-                $grabber_data->{$grabber}->{latest_data_seen} = $t2->epoch if ($t2->epoch > $grabber_data->{$grabber}->{latest_data_seen});
+                $plugin_data->{$grabber}->{earliest_data_seen} = $t1->epoch if (!defined $plugin_data->{$grabber}->{earliest_data_seen});
+                $plugin_data->{$grabber}->{earliest_data_seen} = $t1->epoch if ($t1->epoch < $plugin_data->{$grabber}->{earliest_data_seen});
+                $plugin_data->{$grabber}->{latest_data_seen} = $t2->epoch if (!defined $plugin_data->{$grabber}->{latest_data_seen});
+                $plugin_data->{$grabber}->{latest_data_seen} = $t2->epoch if ($t2->epoch > $plugin_data->{$grabber}->{latest_data_seen});
 
                 # store channel-specific stats
@@ -299 +314 @@
 
         # print some stats about what we saw!
-        my $earlist_data_seen = localtime($grabber_data->{$grabber}->{earliest_data_seen});
-        my $latest_data_seen = localtime($grabber_data->{$grabber}->{latest_data_seen});
-        printf "SHEPHERD: Grabber '%s' returned data for %d channels (%d programmes, %dd%02dh%02dm%02ds duration, earliest %s, latest %s)\n",
-            $grabber, $seen_channels_with_data, $grabber_data->{$grabber}->{programmes},
-            int($grabber_data->{$grabber}->{total_duration} / 86400),           # days
-            int(($grabber_data->{$grabber}->{total_duration} % 86400) / 3600),  # hours
-            int(($grabber_data->{$grabber}->{total_duration} % 3600) / 60),     # mins
-            int($grabber_data->{$grabber}->{total_duration} % 60),              # sec
-            $earlist_data_seen, $latest_data_seen;
+        printf "SHEPHERD: Grabber '%s' returned data for %d channels, %d programmes, %dd%02dh%02dm%02ds duration, %s%s\n",
+            $grabber, $seen_channels_with_data, $plugin_data->{$grabber}->{programmes},
+            int($plugin_data->{$grabber}->{total_duration} / 86400),            # days
+            int(($plugin_data->{$grabber}->{total_duration} % 86400) / 3600),   # hours
+            int(($plugin_data->{$grabber}->{total_duration} % 3600) / 60),      # mins
+            int($plugin_data->{$grabber}->{total_duration} % 60),               # sec
+            (defined $plugin_data->{$grabber}->{earliest_data_seen} ? (strftime "%a %e %b %H:%M - ", localtime($plugin_data->{$grabber}->{earliest_data_seen})) : 'no data'),
+            (defined $plugin_data->{$grabber}->{latest_data_seen} ? (strftime "%a %e %b %H:%M", localtime($plugin_data->{$grabber}->{latest_data_seen})) : '');
+        $plugin_data->{$grabber}->{laststatus} = sprintf "%dch/%dpr/%dhrs %s-%s",
+            $seen_channels_with_data, $plugin_data->{$grabber}->{programmes},
+            int($plugin_data->{$grabber}->{total_duration} / 3600),
+            (defined $plugin_data->{$grabber}->{earliest_data_seen} ? (strftime "%a%d%b%H:%M", localtime($plugin_data->{$grabber}->{earliest_data_seen})) : 'no'),
+            (defined $plugin_data->{$grabber}->{latest_data_seen} ? (strftime "%a%d%b%H:%M", localtime($plugin_data->{$grabber}->{latest_data_seen})) : 'data');
+
     } else {
-        printf "WARNING: Grabber %s didn't seem to return any valid XMLTV!\n",$grabber;
-        delete $grabber_data->{$grabber}->{valid};
+        printf "WARNING: Plugin %s didn't seem to return any valid XMLTV!\n",$grabber;
+        delete $plugin_data->{$grabber}->{valid};
     }
 }
@@ -317 +337 @@
 # analyze grabber data - do we have all the data we want?
 # returns 1 if we need more data, 0 if we have all we want
-sub analyze_grabber_data
-{
+sub analyze_plugin_data
+{
+    my $threshold = shift;
     my $retval = 0; # until proven otherwise
     my $total_data_percent = 0, my $total_channels = 0;
@@ -329 +350 @@
             my $data_in_channel = 0;
             for my $slotnum (0..($num_timeslots-1)) {
-                $data_in_channel++ if ($channel_data->{$ch}->{timeslots}[$slotnum] > 0);
+                $data_in_channel++ if ((defined $channel_data->{$ch}->{timeslots}[$slotnum]) && ($channel_data->{$ch}->{timeslots}[$slotnum] > 0));
             }
 
             # do we have enough data for this channel?
             my $data_in_channel_percent = $data_in_channel / ($num_timeslots-1) * 100;
-            if ($data_in_channel_percent >= $channel_ok_threshold_percent) {
+            if ($data_in_channel_percent >= $threshold) {
                 $statusstring .= sprintf "%s: %0.1f%% [complete], ",$ch,$data_in_channel_percent;
             } else {
@@ -381 +402 @@
 
 # -----------------------------------------
+# Subs: Postprocessing
+# -----------------------------------------
+
+sub postprocess_data
+{
+    # for our first postprocessor, we feed it ALL of the XMLTV files we have
+    # as each postprocessor runs, we feed in the output from the previous one
+    # Shepherd checks the "completeness" of the data that comes out of a postprocessor & automatically
+    # reverts back to the previous postprocessor if it was shown to be bad
+
+    # first time around: feed in $input_postprocess_files
+    my $need_more_data;
+
+    foreach my $postprocessor (sort { $postprocessors->{$a} <=> $postprocessors->{$b} } keys %$postprocessors) {
+        next if ($postprocessors->{$postprocessor}->{disabled});
+
+        $postprocessors->{$postprocessor}->{lastdata} = time;
+        $postprocessors->{$postprocessor}->{laststatus} = "unknown";
+
+        printf "SHEPHERD: Using postprocessor: %s\n",$postprocessor;
+
+        my $output = "$POSTPROCESSOR_DIR/$postprocessor/output.xmltv";
+        my $comm = "$POSTPROCESSOR_DIR/$postprocessor/$postprocessor " .
+                   "--region $region " .
+                   "--channels_file $channels_file " .
+                   "--output $output";
+        $comm .= " --days $days" if ($days);
+        $comm .= " --offset $opt->{offset}" if ($opt->{offset});
+        $comm .= " --debug" if ($debug);
+        $comm .= " @ARGV" if (@ARGV);
+        $comm .= " $input_postprocess_files";
+        print "SHEPHERD: Excuting command: $comm\n";
+
+        chdir "$POSTPROCESSOR_DIR/$postprocessor/";
+        system($comm);
+        chdir $CWD;
+
+        #
+        # soak up the data we just collected and check it
+        # YES - these are the SAME routines we used in the previous 'grabber' phase
+        # but the difference here is that we clear out our 'channel_data' beforehand
+        # so we can independently analyze the impact of this postprocessor.
+        # if it clearly returns bad data, don't use that data (go back one step) and
+        # flag the postprocessor as having failed.  after 3 consecutive failures, disable it
+        #
+
+        # clear out channel_data
+        foreach my $ch (keys %{$channels}) {
+            delete $channel_data->{$ch};
+        }
+
+        # process and analyze it!
+        &soak_up_data($postprocessor, $output);
+        $need_more_data = &analyze_plugin_data($postprocessor_ok_threshold_percent);
+
+        $postprocessors->{$postprocessor}->{laststatus} = $plugin_data->{$postprocessor}->{laststatus};
+
+        if (($need_more_data) && (!$insufficient_grabber_data)) {
+            # urgh.  this postprocessor did a bad bad thing ...
+            printf "SHEPHERD: XML data from postprocessor %s rejected, using XML from previous stage\n",$postprocessor;
+
+            if (defined $postprocessors->{$postprocessor}->{conescutive_failures}) {
+                $postprocessors->{$postprocessor}->{conescutive_failures}++;
+            } else {
+                $postprocessors->{$postprocessor}->{conescutive_failures} = 1;
+            }
+            printf "SHEPHERD: Postprocessor \"%s\" has now failed %d times in a row.  %d more and it will be automatically disabled.\n",
+                $postprocessor,
+                $postprocessors->{$postprocessor}->{conescutive_failures},
+                $postprocessor_disable_failure_threshold;
+
+            if ($postprocessors->{$postprocessor}->{conescutive_failures} >= $postprocessor_disable_failure_threshold) {
+                printf "SHEPHERD: Disabling Postprocessor \"%s\".\n",$postprocessor;
+                $postprocessors->{$postprocessor}->{disabled} = 1;
+            }
+        } else {
+            # accept what this postprocessor did to our output ...
+            printf "SHEPHERD: accepting output from postprocessor %s, feeding it into next stage\n",$postprocessor;
+            $input_postprocess_files = $output;
+            delete $postprocessors->{$postprocessor}->{conescutive_failures} if (defined $postprocessors->{$postprocessor}->{conescutive_failures});
+        }
+    }
+}
+
+
+sub output_data
+{
+    # $input_postprocess_files (hopefully just one file now) contains our final output
+    # send it to whereever --output told us to!
+
+    if ($opt->{output}) {
+        open(F,">$opt->{output}") || die "could not open outputfile $opt->{output} for writing: $!\n";
+    }
+
+    foreach my $infile (split(/ /,$input_postprocess_files)) {
+        if (!(open(INFILE,"<$infile"))) {
+            printf "WARNING: could not open input file \"%s\": %s\n", $infile, $!;
+            printf "Output XMLTV data may be damanged as a result!\n";
+        } else {
+            while (<INFILE>) {
+                if ($opt->{output}) {
+                    print F $_ if ($opt->{output});
+                } else {
+                    print $_;
+                }
+            }
+            close(INFILE);
+        }
+    }
+    close(F) if ($opt->{output});
+}
+
+# -----------------------------------------
 # Subs: Updates & Installations
 # -----------------------------------------
@@ -395 +529 @@
 
     my %glist = %$grabbers;
-    while ($data =~ /(.*):(.*)/g)
-    {
-        update_component($1, $2);
-        delete $glist{$1};
-    }
-    foreach (keys %glist)
-    {
-        unless ($grabbers->{$_}->{disabled})
-        {
+    my %plist = %$postprocessors;
+    while ($data =~ /(.*):(.*):(.*)/g)
+    {
+        my ($proggy, $latestversion, $progtype) = ($1,$2,$3);
+        update_component($proggy, $latestversion, $progtype);
+        delete $glist{$proggy} if ($progtype eq "grabber");
+        delete $plist{$proggy} if ($progtype eq "postprocessor");
+    }
+
+    # work out what grabbers disappeared (if any)
+    foreach (keys %glist) {
+        unless ($grabbers->{$_}->{disabled}) {
             print "\nDeleted grabber: $_.\n";
-            disable($_);
+            disable($_,"grabber");
+            $made_changes = 1;
+        }
+    }
+
+    # work out what postprocessors disappeared (if any)
+    foreach (keys %plist) {
+        unless ($postprocessors->{$_}->{disabled}) {
+            print "\nDeleted Postprocessor: $_.\n";
+            disable($_,"postprocessor");
+            $made_changes = 1;
         }
     }
@@ -412 +559 @@
 sub update_component
 {
-    my ($proggy, $latestversion) = @_;
+    my ($proggy, $latestversion, $progtype) = @_;
 
     print "\n";
 
-    if ($proggy eq $progname)
-    {
-        if(! -e "$CWD/$progname")
-        {
+    # handle new installs..
+    if (($$proggy eq $progname) && ($progtype eq "shepherd")) {
+        # shepherd itself..
+        if(! -e "$CWD/$progname") {
             print "Missing: $CWD/$progname\n";
-            install($progname, $latestversion);
+            install($progname, $latestversion, $progtype);
             return;
         }
-    }
-    else
-    {
-        if (!$grabbers->{$proggy} or ! -e "$GRABBER_DIR/$proggy/$proggy")
-        {
+    } elsif ($progtype eq "grabber") {
+        if (!$grabbers->{$proggy} or ! -e "$GRABBER_DIR/$proggy/$proggy") {
             print "New grabber: $proggy.\n";
-            install($proggy, $latestversion);
+            install($proggy, $latestversion, $progtype);
             return;
         }
-        if ($grabbers->{$proggy}->{disabled})
-        {
-            print "Warning: $proggy disabled by config file.\n";
-        }
-    }
-
-    # Compare versions
-    my $ver = ($proggy eq $progname ? $version : $grabbers->{$proggy}->{ver});
+        print "Warning: grabber $proggy disabled by config file.\n" if ($grabbers->{$proggy}->{disabled});
+    } elsif ($progtype eq "postprocessor") {
+        if (!$postprocessors->{$proggy} or ! -e "$POSTPROCESSOR_DIR/$proggy/$proggy") {
+            print "New postprocessor: $proggy.\n";
+            install($proggy, $latestversion, $progtype);
+            return;
+        }
+        print "Warning: postprocessor $proggy disabled by config file.\n" if ($postprocessors->{$proggy}->{disabled});
+    }
+
+    # upgrade/downgrades
+    my $ver;
+    if ($progtype eq "grabber") {
+        $ver = ($proggy eq $progname ? $version : $grabbers->{$proggy}->{ver});
+    } elsif ($progtype eq "postprocessor") {
+        $ver = ($proggy eq $progname ? $version : $postprocessors->{$proggy}->{ver});
+    } elsif (($$proggy eq $progname) && ($progtype eq "shepherd")) {
+        $ver = $version;
+    } else {
+        print "Warning: unknown type of programme: prog '$proggy' progtype '$progtype' not installed.\n";
+        return;
+    }
 
     my $result = versioncmp($ver, $latestversion);
-    if ($result == -1)
-    {
+    if ($result == -1) {
         print "Upgrading $proggy from v$ver to v$latestversion.\n";
-    }
-    elsif ($result == 1)
-    {
+    } elsif ($result == 1) {
         print "Downgrading $proggy from v$ver to v$latestversion.\n";
-    }
-    else
-    {
+    } else {
         print "Already have latest version of $proggy: v$ver.\n";
         return;
     }
-    install($proggy, $latestversion);
+    install($proggy, $latestversion, $progtype);
 }
 
 sub install
 {
-    my ($proggy, $latestversion) = @_;
+    my ($proggy, $latestversion, $progtype) = @_;
 
     print "Downloading $proggy v$latestversion.\n";
@@ -469 +622 @@
     my $ver = $version;
 
-    if ($proggy ne $progname)
-    {
-        -d $GRABBER_DIR or mkdir $GRABBER_DIR
-            or die "Cannot create directory $GRABBER_DIR: $!";
-
+    if (($$proggy eq $progname) && ($progtype eq "shepherd")) {
+        $rdir = $HOME;
+        $ldir = $CWD;
+        $ver = $version;
+    } elsif ($progtype eq "grabber") {
+        $rdir = $HOME . "/grabbers";
         $ldir = "$GRABBER_DIR/$proggy";
-        -d $ldir or mkdir $ldir or die "Cannot create directory $ldir: $!";
-        $rdir = "$rdir/grabbers";
         $ver = $grabbers->{$proggy}->{ver};
-    }
+        -d $GRABBER_DIR or mkdir $GRABBER_DIR or die "Cannot create directory $GRABBER_DIR: $!";
+    } elsif ($progtype eq "postprocessor") {
+        $rdir = $HOME . "/postprocessors";
+        $ldir = "$POSTPROCESSOR_DIR/$proggy";
+        $ver = $postprocessors->{$proggy}->{ver};
+        -d $POSTPROCESSOR_DIR or mkdir $POSTPROCESSOR_DIR or die "Cannot create directory $POSTPROCESSOR_DIR: $!";
+    } else {
+        print "Warning: unknown type of programme: prog '$proggy' progtype '$progtype' not installed.\n";
+        return;
+    }
+    -d $ldir or mkdir $ldir or die "Cannot create directory $ldir: $!";
 
     my $newfile = "$ldir/$proggy-$latestversion";
-    
     my $rc = LWP::Simple::getstore("$rdir/$proggy-$latestversion", $newfile);
 
@@ -493 +654 @@
     system('chmod u+x ' . $newfile);
 
-    -d $ARCHIVE_DIR or mkdir $ARCHIVE_DIR
-      or die "Cannot create directory $ARCHIVE_DIR: $!";
+    -d $ARCHIVE_DIR or mkdir $ARCHIVE_DIR or die "Cannot create directory $ARCHIVE_DIR: $!";
 
     if (-e "$ldir/$proggy")
@@ -504 +664 @@
     print "Installed $proggy v$latestversion.\n" if ($debug);
 
-    if ($proggy eq $progname)
-    {
+    # if the update was for shepherd itself, restart it
+    if (($$proggy eq $progname) && ($progtype eq "shepherd")) {
         print "\n*** Restarting ***\n\n";
         exec("$ldir/$proggy");
@@ -512 +672 @@
 
     print "Testing $proggy...\n" if ($debug);
-    my $result = test_grabber($proggy);
-
-    # Update grabbers list
-    my $disabled = $grabbers->{$proggy}->{disabled};
-    $grabbers->{$proggy} = 
-        { 
-            'ver' => $latestversion, 
-            'disabled' => $disabled,
-            'ready' => $result
-        };
-}
-
-sub test_grabber
-{
-    my $proggy = shift;
-
-    chdir("$GRABBER_DIR/$proggy");
-    system("$GRABBER_DIR/$proggy/$proggy --ready");
+    my $result = test_proggy($ldir,"$ldir/$proggy");
+
+    if ($progtype eq "grabber") {
+        $grabbers->{$proggy}->{ver} = $latestversion;
+        $grabbers->{$proggy}->{ready} = $result;
+        $grabbers->{$proggy}->{laststatus} = sprintf "updated to %s on %s", $latestversion, (strftime "%a%d%b%y",localtime(time));
+    } elsif ($progtype eq "postprocessor") {
+        $postprocessors->{$proggy}->{ver} = $latestversion;
+        $postprocessors->{$proggy}->{ready} = $result;
+        $postprocessors->{$proggy}->{laststatus} = sprintf "updated to %s on %s", $latestversion, (strftime "%a%d%b%y",localtime(time));
+    }
+
+    $made_changes = 1;
+}
+
+sub test_proggy
+{
+    my ($testdir,$proggyexec) = @_;
+
+    chdir($testdir);
+    system("$proggyexec --ready");
     chdir ($CWD);
 
@@ -535 +698 @@
     print "Return value: $result\n" if ($debug);
 
-    if ($result)
-    {
-        print "\nGrabber $proggy did not exit cleanly!\n" .
-              "It may require configuration.\n\n";
-    }
-
+    print "\nprogramme $proggyexec did not exit cleanly!\n" .
+         "It may require configuration.\n\n" if ($result);
     return !$result;
 }
@@ -548 +707 @@
     my $proggy = shift;
 
+    # confirm it exists first
+    if ((!$grabbers->{$proggy}) && (!$postprocessors->{$proggy})) {
+        printf "No such grabber/postprocessor: \"%s\".\n",$proggy;
+        return;
+    }
     print "Enabling $proggy.\n";
-    if (!$grabbers->{$proggy})
-    {
-        print "No such grabber: \"$proggy\".\n";
-    }
-    else
-    {
-        delete $grabbers->{$proggy}->{disabled};
-    }
+
+    delete $grabbers->{$proggy}->{disabled} if ($grabbers->{$proggy});
+    delete $postprocessors->{$proggy}->{disabled} if ($postprocessors->{$proggy});
+    $made_changes = 1;
 }
 
@@ -563 +723 @@
     my $proggy = shift;
 
+    # confirm it exists first
+    if ((!$grabbers->{$proggy}) && (!$postprocessors->{$proggy})) {
+        printf "No such grabber/postprocessor: \"%s\".\n",$proggy;
+        return;
+    }
     print "Disabling $proggy.\n";
-    $grabbers->{$proggy}->{disabled} = 1;
+
+    $grabbers->{$proggy}->{disabled} = 1 if ($grabbers->{$proggy});
+    $postprocessors->{$proggy}->{disabled} = 1 if ($postprocessors->{$proggy});
+    $made_changes = 1;
 }
 
 sub set_order
 {
-    my $order = shift;
-    my $all_ok = 1;
-
-    # first check that user supplied a valid list of grabbers
-    if ($order) {
-        foreach my $proggy (split(/,/,$order)) {
-            next if !$proggy;
-            if (!$grabbers->{$proggy}) {
-                $all_ok = 0;
-                print "Invalid grabber: \"$proggy\".\n";
-            }
-        }
-    }
-    if ($all_ok) {
-        # if list was ok then first reset current order to zero
-        foreach my $proggy (keys %$grabbers) {
-            $grabbers->{$proggy}->{order} = 0;
-        }
-
-        # and now set order
-        my $order_num = 1;
-        if ($order) {
-            foreach my $proggy (split(/,/,$order)) {
+    my ($quiet,$order) = @_;
+    $pref_order = $order if ($order);
+
+    # reset current order to zero
+    foreach my $proggy (keys %$grabbers) {
+        $grabbers->{$proggy}->{order} = 0;
+    }
+
+    # and now set order
+    my $order_num = 1;
+    if ($pref_order) {
+        foreach my $proggy (split(/,/,$pref_order)) {
+            if (defined $grabbers->{$proggy}) {
                 $grabbers->{$proggy}->{order} = $order_num;
                 $order_num++;
             }
         }
-
-        # set order of any grabbers not specified in a random manner
-        foreach my $proggy (sort keys %$grabbers) {
-            if ($grabbers->{$proggy}->{order} == 0) {
-                $grabbers->{$proggy}->{order} = $order_num+int(rand(100));
-            }
-        }
-
-        # .. and finally normalize the order (& show the user the order we chose)
-        print "Grabber order set as follows:\n";
-        $order_num = 0;
-        $opt->{setorder} = "";
-        foreach my $proggy (sort { $grabbers->{$a}->{order} <=> $grabbers->{$b}->{order} } keys %$grabbers) {
-            $order_num++;
-            $grabbers->{$proggy}->{order} = $order_num;
-            $opt->{setorder} .= "$proggy" . ",";
-            printf " #%d. %s%s\n",$grabbers->{$proggy}->{order},$proggy,($grabbers->{$proggy}->{disabled} ? " [disabled]" : "");
-        }
-    }
+    }
+
+    # set order of any grabbers not specified in a random manner
+    foreach my $proggy (sort keys %$grabbers) {
+        if ((!defined $grabbers->{$proggy}->{order}) || ($grabbers->{$proggy}->{order} == 0)) {
+            $grabbers->{$proggy}->{order} = $order_num+int(rand(1000));
+        }
+    }
+
+    # .. and finally normalize the order (& show the user the order we chose)
+    print "Grabber order set as follows:\n" unless $quiet;
+    $order_num = 0;
+    foreach my $proggy (sort { $grabbers->{$a}->{order} <=> $grabbers->{$b}->{order} } keys %$grabbers) {
+        $order_num++;
+        $grabbers->{$proggy}->{order} = $order_num;
+        printf " #%d. %s%s\n",$grabbers->{$proggy}->{order},$proggy,($grabbers->{$proggy}->{disabled} ? " [disabled]" : "") unless $quiet;
+    }
+
+    $made_changes = 1;
 }
 
@@ -620 +778 @@
 {
     my $result;
-    foreach (keys %$grabbers)
-    {
-        $result = test_grabber($_);
-        print "Grabber $_: " . 
-              ($result ? "OK" : "Failed") . "\n\n";
-        if (!$result ne !$grabbers->{$_}->{ready})
-        {
-            $grabbers->{$_}->{ready} = $result;
+    foreach my $proggy (keys %$grabbers) {
+        $result = test_proggy("$GRABBER_DIR/$proggy","$GRABBER_DIR/$proggy/$proggy");
+        printf "Grabber %s: %s\n",$proggy,($result ? "OK" : "Failed");
+        if (!$result ne !$grabbers->{$proggy}->{ready}) {
+            $grabbers->{$proggy}->{ready} = $result;
+            $made_changes = 1;
+        }
+    }
+
+    foreach my $proggy (keys %$postprocessors) {
+        $result = test_proggy("$POSTPROCESSOR_DIR/$proggy","$POSTPROCESSOR_DIR/$proggy/$proggy");
+        printf "Postprocessor %s: %s\n",$proggy,($result ? "OK" : "Failed");
+        if (!$result ne !$postprocessors->{$proggy}->{ready}) {
+            $postprocessors->{$proggy}->{ready} = $result;
+            $made_changes = 1;
         }
     }
@@ -651 +816 @@
         # at least one 'order' was missing .. we need to put it in!
         printf "Legacy shepherd.conf file didn't contain any grabber order! Automatically updating using a random order, use --setorder to manually set this if you care.\n";
-        &set_order();
-        $opt->{setorder} = "";
+        &set_order(1);
     }
 }
@@ -691 +855 @@
     open(CONF, ">$config_file") or die "cannot write to $config_file: $!";
     print CONF Data::Dumper->Dump(
-        [$region,  $grabbers ],
-        ["region", "grabbers" ]);
+        [$region,  $pref_order,   $grabbers, $postprocessors  ],
+        ["region", "pref_order", "grabbers", "postprocessors" ]);
     close CONF;
     print "\nUpdated configuration file $config_file.\n" if ($debug);
@@ -710 +874 @@
               'help'            => \$opt->{help},
               'configure'       => \$opt->{configure},
+              'output'          => \$opt->{output},
               'debug'           => \$debug);
 }
@@ -802 +967 @@
 
     print "\nRandomly selecting grabber order.\n\n";
-    set_order();
+    set_order(0);
 
     show_channels();
@@ -882 +1047 @@
 sub status
 {
-    print " Grabber                     Version  Enabled  Ready    Last Data\n" .
-          " -----------------------------------------------------------------\n";
-    my $star;
-    foreach (sort { $grabbers->{$a}->{order} <=> $grabbers->{$b}->{order} } keys %$grabbers)
-    {
+    print " Grabber           Version Enabled Ready Last Run   Status\n" .
+          " ----------------- ------- ------- ----- ---------- ---------------------------\n";
+    foreach (sort { $grabbers->{$a}->{order} <=> $grabbers->{$b}->{order} } keys %$grabbers) {
         my $h = $grabbers->{$_};
-        printf  " %-26s %8s %5s %7s %10s\n",
+        printf  " %-16s %8s %4s %6s  %11s %s\n",
                 "$h->{order}. $_",
                 $h->{ver},
                 $h->{disabled} ? '' : 'Y',
                 $h->{ready} ? 'Y' : '',
-                $h->{lastdata} ? localtime($h->{lastdata}) : '';
-    }
-
-    printf "Grabbers shown in order of preference.\n";
+                $h->{lastdata} ? (strftime "%a%d%b%y", localtime($h->{lastdata})) : '',
+                $h->{laststatus} ? $h->{laststatus} : '';
+    }
+    printf "Grabbers shown in order of preference.\n\n";
+
+    print " Postprocessor     Version Enabled Ready Last Run   Status\n" .
+          " ----------------- ------- ------- ----- ---------- ---------------------------\n";
+    foreach (sort { $postprocessors->{$a} <=> $postprocessors->{$b} } keys %$postprocessors) {
+        my $h = $postprocessors->{$_};
+        printf  " %-16s %8s %4s %6s  %11s %s\n",
+                $_,
+                $h->{ver},
+                $h->{disabled} ? '' : 'Y',
+                $h->{ready} ? 'Y' : '',
+                $h->{lastdata} ? (strftime "%a%d%b%y", localtime($h->{lastdata})) : '',
+                $h->{laststatus} ? $h->{laststatus} : '';
+    }
+    printf "Postprocessors shown in order of execution.\n\n";
 }
 
@@ -912 +1089 @@
 
     --setorder <s>        Set order of grabbers to <s> (comma-seperated list of grabbers)
-    --disable <s>         Don't ever use grabber <s>
+
+    --disable <s>         Don't ever use grabber/postprocessor <s>
     --enable <s>          Okay, maybe use it again then
-    --uninstall <s>       Remove a disabled grabber
+    --uninstall <s>       Remove a disabled grabber/postprocessor
 
     --noupdate            Do not attempt to update before running
     --update              Update only; do not grab data
 
-    --check               Check status of all grabbers
+    --check               Check status of all grabbers and postprocessors
 };
     exit 0;