Changeset 38

Timestamp:

10/08/06 06:08:28 (7 years ago)

Author:

max

Message:

Combined $grabbers and $postprocessors into $components to cut down
on redundant code.

Files:

• shepherd (modified) (36 diffs)

shepherd

@@ -23 +23 @@
 # 0.2.7   : Changed online file structure
 # 0.2.8   : Integrated reconciler
+#
+#   have a type, and query_grabbers() and query_postprocessors() returning
+#   lists of each for easy iteration.
 
 BEGIN { *CORE::GLOBAL::die = \&my_die; }
@@ -55 +58 @@
 chdir($CWD);
 
-my $GRABBER_DIR = "$CWD/grabbers";
-my $POSTPROCESSOR_DIR = "$CWD/postprocessors";
 my $ARCHIVE_DIR = "$CWD/archive";
 
@@ -112 +113 @@
 my $debug = 1;
 my $recdebug = 0;
-my $grabbers = { };
+my $components = { };
 my $gscore;
-my $preferred; # obsolete but may still exist in shepherd.conf
 my $region;
 my $channels;
@@ -122 +122 @@
 
 # postprocessing
-my $postprocessors = { };
 my $title_translation_table = { };
 my $langs = [ 'en' ];
@@ -132 +131 @@
 my $insufficient_grabber_data = 0;
 
+# OBSOLETE: will be removed
+my $grabbers;
+my $postprocessors;
+my $preferred;
+
 # ---------------------------------------------------------------------------
 # --- Setup
@@ -253 +257 @@
         $used_grabbers++;
 
-        $grabbers->{$grabber}->{lastdata} = time;
-        $grabbers->{$grabber}->{laststatus} = "unknown";
+        $components->{$grabber}->{lastdata} = time;
+        $components->{$grabber}->{laststatus} = "unknown";
 
         printf "\nSHEPHERD: Using grabber: (%d) %s\n", $used_grabbers, $grabber;
 
-        my $output = "$GRABBER_DIR/$grabber/output.xmltv";
-
-        my $comm = "$GRABBER_DIR/$grabber/$grabber " .
+        my $output = "$CWD/grabbers/$grabber/output.xmltv";
+
+        my $comm = "$CWD/grabbers/$grabber/$grabber " .
                    "--region $region " .
                    "--channels_file $channels_file " .
@@ -277 +281 @@
         } else {
             print "SHEPHERD: Excuting command: $comm\n";
-            chdir "$GRABBER_DIR/$grabber/";
+            chdir "$CWD/grabbers/$grabber/";
             system($comm);
             chdir $CWD;
@@ -284 +288 @@
         # soak up the data we just collected
         &soak_up_data($grabber, $output, "grabber");
-        $grabbers->{$grabber}->{laststatus} = $plugin_data->{$grabber}->{laststatus};
+        $components->{$grabber}->{laststatus} = $plugin_data->{$grabber}->{laststatus};
 
         # check to see if we have all the data we want
@@ -311 +315 @@
     unless (defined $gscore)
     {
-        foreach (keys %$grabbers)
+        foreach (query_grabbers())
         {
-            unless ($grabbers->{$_}->{disabled})
+            unless ($components->{$_}->{disabled})
             {
                 $gscore->{$_} = 0;
@@ -375 +379 @@
 #           print "$grabber vs $ch:$day: " . ($result ? "OK" : "no") . "\n";
         }
-        $niceness = $grabbers->{$grabber}->{config}->{niceness};
+        $niceness = $components->{$grabber}->{config}->{niceness};
         unless ($niceness)
         {
@@ -382 +386 @@
             $niceness = 5;
         }
-        $granularity = $grabbers->{$grabber}->{config}->{granularity};
+        $granularity = $components->{$grabber}->{config}->{granularity};
         unless (defined $granularity)
         {
@@ -406 +410 @@
     my ($grabber, $ch) = @_;
 
-    my $channels_supported = $grabbers->{$grabber}->{config}->{channels};
+    my $channels_supported = $components->{$grabber}->{config}->{channels};
     unless (defined $channels_supported)
     {
@@ -430 +434 @@
     my ($grabber, $day) = @_;
 
-    my $days_supported = $grabbers->{$grabber}->{config}->{max_days};
+    my $days_supported = $components->{$grabber}->{config}->{max_days};
     unless ($days_supported)
     {
@@ -674 +678 @@
 
     printf "Preference for whose data we prefer as follows:\n";
-    foreach my $proggy (sort { $grabbers->{$a}->{order} <=> $grabbers->{$b}->{order} } keys %$grabbers) {
-        if ((!$grabbers->{$proggy}->{disabled}) && ($plugin_data->{$proggy}) && ($plugin_data->{$proggy}->{valid})) {
+    foreach my $proggy (sort { $components->{$a}->{order} <=> $components->{$b}->{order} } query_grabbers()) {
+        if ((!$components->{$proggy}->{disabled}) && ($plugin_data->{$proggy}) && ($plugin_data->{$proggy}->{valid})) {
             $grabber_order[$num_grabbers] = $proggy;
             my $orig_prog = $plugin_data->{$proggy}->{xmltv}->[3];
@@ -1045 +1049 @@
     printf "\nPostprocessing stage:\n";
 
-    foreach my $postprocessor (sort { $postprocessors->{$a} <=> $postprocessors->{$b} } keys %$postprocessors) {
-        next if ($postprocessors->{$postprocessor}->{disabled});
-
-        $postprocessors->{$postprocessor}->{lastdata} = time;
-        $postprocessors->{$postprocessor}->{laststatus} = "unknown";
+    foreach my $postprocessor (sort { $components->{$a} <=> $components->{$b} } query_postprocessors()) {
+        next if ($components->{$postprocessor}->{disabled});
+
+        $components->{$postprocessor}->{lastdata} = time;
+        $components->{$postprocessor}->{laststatus} = "unknown";
 
         printf "\nSHEPHERD: Using postprocessor: %s\n",$postprocessor;
 
-        my $output = "$POSTPROCESSOR_DIR/$postprocessor/output.xmltv";
-        my $comm = "$POSTPROCESSOR_DIR/$postprocessor/$postprocessor " .
+        my $output = "$CWD/postprocessors/$postprocessor/output.xmltv";
+        my $comm = "$CWD/postprocessors/$postprocessor/$postprocessor " .
                    "--region $region " .
                    "--channels_file $channels_file " .
@@ -1065 +1069 @@
         print "SHEPHERD: Excuting command: $comm\n";
 
-        chdir "$POSTPROCESSOR_DIR/$postprocessor/";
+        chdir "$CWD/postprocessors/$postprocessor/";
         system($comm);
         chdir $CWD;
@@ -1087 +1091 @@
         $need_more_data = &analyze_plugin_data($postprocessor_ok_threshold_percent, "POSTPROCESSOR");
 
-        $postprocessors->{$postprocessor}->{laststatus} = $plugin_data->{$postprocessor}->{laststatus};
+        $components->{$postprocessor}->{laststatus} = $plugin_data->{$postprocessor}->{laststatus};
 
         if (($need_more_data) && (!$insufficient_grabber_data)) {
@@ -1093 +1097 @@
             printf "SHEPHERD: XML data from postprocessor %s rejected, using XML from previous stage\n",$postprocessor;
 
-            if (defined $postprocessors->{$postprocessor}->{conescutive_failures}) {
-                $postprocessors->{$postprocessor}->{conescutive_failures}++;
+            if (defined $components->{$postprocessor}->{conescutive_failures}) {
+                $components->{$postprocessor}->{conescutive_failures}++;
             } else {
-                $postprocessors->{$postprocessor}->{conescutive_failures} = 1;
+                $components->{$postprocessor}->{conescutive_failures} = 1;
             }
             printf "SHEPHERD: Postprocessor \"%s\" has now failed %d times in a row.  %d more and it will be automatically disabled.\n",
                 $postprocessor,
-                $postprocessors->{$postprocessor}->{conescutive_failures},
-                ($postprocessor_disable_failure_threshold - $postprocessors->{$postprocessor}->{conescutive_failures});
-
-            if ($postprocessors->{$postprocessor}->{conescutive_failures} >= $postprocessor_disable_failure_threshold) {
+                $components->{$postprocessor}->{conescutive_failures},
+                ($postprocessor_disable_failure_threshold - $components->{$postprocessor}->{conescutive_failures});
+
+            if ($components->{$postprocessor}->{conescutive_failures} >= $postprocessor_disable_failure_threshold) {
                 printf "SHEPHERD: Disabling Postprocessor \"%s\".\n",$postprocessor;
-                $postprocessors->{$postprocessor}->{disabled} = 1;
+                $components->{$postprocessor}->{disabled} = 1;
             }
         } else {
@@ -1111 +1115 @@
             printf "SHEPHERD: accepting output from postprocessor %s, feeding it into next stage\n",$postprocessor;
             $input_postprocess_file = $output;
-            delete $postprocessors->{$postprocessor}->{conescutive_failures} if (defined $postprocessors->{$postprocessor}->{conescutive_failures});
+            delete $components->{$postprocessor}->{conescutive_failures} if (defined $components->{$postprocessor}->{conescutive_failures});
         }
     }
@@ -1164 +1168 @@
     return if (!$data);
 
-    my %glist = %$grabbers;
-    my %plist = %$postprocessors;
+    my %clist = %$components;
     while ($data =~ /(.*):(.*):(.*)/g)
     {
         my ($proggy, $latestversion, $progtype) = ($1,$2,$3);
         update_component($proggy, $latestversion, $progtype);
-        delete $glist{$proggy} if ($progtype eq "grabber");
-        delete $plist{$proggy} if ($progtype eq "postprocessor");
-    }
-
-    # work out what grabbers disappeared (if any)
-    foreach (keys %glist) {
-        unless ($grabbers->{$_}->{disabled}) {
-            print "\nDeleted grabber: $_.\n";
-            disable($_,"grabber");
+        delete $clist{$proggy};
+    }
+
+    # work out what components disappeared (if any)
+    foreach (keys %clist) {
+        unless ($components->{$_}->{disabled}) {
+            print "\nDeleted component: $_.\n";
+            disable($_);
             $made_changes = 1;
         }
     }
 
-    # work out what postprocessors disappeared (if any)
-    foreach (keys %plist) {
-        unless ($postprocessors->{$_}->{disabled}) {
-            print "\nDeleted Postprocessor: $_.\n";
-            disable($_,"postprocessor");
-            $made_changes = 1;
-        }
-    }
 }
 
@@ -1198 +1192 @@
 
     # handle new installs..
-    if (($proggy eq $progname) && ($progtype eq "shepherd")) {
-        # shepherd itself..
+    if ($progtype eq "shepherd") {
         if(! -e "$CWD/$progname") {
             print "Missing: $CWD/$progname\n";
@@ -1205 +1198 @@
             return;
         }
-    } elsif ($progtype eq "grabber") {
-        if (!defined $grabbers->{$proggy} or ! -e "$GRABBER_DIR/$proggy/$proggy") {
-            print "New grabber: $proggy.\n";
+    } else {
+        if (!defined $components->{$proggy} or ! -e ($progtype . "s/$proggy/$proggy")) {
+            print "New $progtype: $proggy.\n";
             install($proggy, $latestversion, $progtype);
             return;
         }
-        print "Warning: grabber $proggy disabled by config file.\n" if ($grabbers->{$proggy}->{disabled});
-    } elsif ($progtype eq "postprocessor") {
-        if (!defined $postprocessors->{$proggy} or ! -e "$POSTPROCESSOR_DIR/$proggy/$proggy") {
-            print "New postprocessor: $proggy.\n";
-            install($proggy, $latestversion, $progtype);
-            return;
-        }
-        print "Warning: postprocessor $proggy disabled by config file.\n" if ($postprocessors->{$proggy}->{disabled});
+        if ($components->{$proggy}->{disabled}) {
+            print "Warning: grabber $proggy disabled by config file.\n";
+        }
     }
 
     # upgrade/downgrades
     my $ver;
-    if ($progtype eq "grabber") {
-        $ver = ($proggy eq $progname ? $version : $grabbers->{$proggy}->{ver});
-    } elsif ($progtype eq "postprocessor") {
-        $ver = ($proggy eq $progname ? $version : $postprocessors->{$proggy}->{ver});
-    } elsif (($proggy eq $progname) && ($progtype eq "shepherd")) {
+    if ($progtype eq "shepherd") {
         $ver = $version;
-    } else {
-        print "Warning: unknown type of programme: prog '$proggy' progtype '$progtype' not installed.\n";
-        return;
-    }
+    }
+    else {
+        $ver = $components->{$proggy}->{ver};
+    } 
 
     my $result = versioncmp($ver, $latestversion);
@@ -1260 +1244 @@
     my $ver = "unknown";
 
-    if (($proggy eq $progname) && ($progtype eq "shepherd")) {
+    if ($progtype eq "shepherd") {
         $ver = $version;
-    } elsif ($progtype eq "grabber") {
-        $rdir = "grabbers";
-        $ldir = "$GRABBER_DIR/$proggy";
-        $ver = $grabbers->{$proggy}->{ver} if ((defined $grabbers->{$proggy}) && $grabbers->{$proggy}->{ver});
-        -d $GRABBER_DIR or mkdir $GRABBER_DIR or die "Cannot create directory $GRABBER_DIR: $!";
-    } elsif ($progtype eq "postprocessor") {
-        $rdir = "postprocessors";
-        $ldir = "$POSTPROCESSOR_DIR/$proggy";
-        $ver = $postprocessors->{$proggy}->{ver} if ((defined $postprocessors->{$proggy}) && $postprocessors->{$proggy}->{ver});
-        -d $POSTPROCESSOR_DIR or mkdir $POSTPROCESSOR_DIR or die "Cannot create directory $POSTPROCESSOR_DIR: $!";
     } else {
-        print "Warning: unknown type of programme: prog '$proggy' progtype '$progtype' not installed.\n";
-        return;
+        $rdir = $progtype . "s";
+        $ldir = "$CWD/$progtype" . "s/$proggy";
+        $ver = $components->{$proggy}->{ver} if ((defined $components->{$proggy}) && $components->{$proggy}->{ver});
+        -d ("$CWD/$progtype" . "s") or mkdir ("$CWD/$progtype" . "s") or die "Cannot create directory $CWD/$progtype" . "s: $!";
     }
     -d $ldir or mkdir $ldir or die "Cannot create directory $ldir: $!";
@@ -1304 +1280 @@
 
     # if the update was for shepherd itself, restart it
-    if (($proggy eq $progname) && ($progtype eq "shepherd")) {
+    if ($progtype eq "shepherd") {
         print "\n*** Restarting ***\n\n";
         exec("$ldir/$proggy");
@@ -1313 +1289 @@
     my $result = test_proggy($ldir,"$ldir/$proggy");
 
-    if ($progtype eq "grabber") {
-        $grabbers->{$proggy}->{ver} = $latestversion;
-        $grabbers->{$proggy}->{ready} = $result;
-        $grabbers->{$proggy}->{laststatus} = sprintf "updated to %s on %s", $latestversion, (strftime "%a%d%b%y",localtime(time));
-    } elsif ($progtype eq "postprocessor") {
-        $postprocessors->{$proggy}->{ver} = $latestversion;
-        $postprocessors->{$proggy}->{ready} = $result;
-        $postprocessors->{$proggy}->{laststatus} = sprintf "updated to %s on %s", $latestversion, (strftime "%a%d%b%y",localtime(time));
-    }
+    $components->{$proggy}->{ver} = $latestversion;
+    $components->{$proggy}->{ready} = $result;
+    $components->{$proggy}->{laststatus} = sprintf "updated to %s on %s", $latestversion, (strftime "%a%d%b%y",localtime(time));
 
     $made_changes = 1;
@@ -1347 +1317 @@
 
     # confirm it exists first
-    if ((!$grabbers->{$proggy}) && (!$postprocessors->{$proggy})) {
-        printf "No such grabber/postprocessor: \"%s\".\n",$proggy;
+    if (!$components->{$proggy}) {
+        printf "No such component: \"%s\".\n",$proggy;
         return;
     }
     print "Enabling $proggy.\n";
 
-    if ($grabbers->{$proggy}) {
-        delete $grabbers->{$proggy}->{disabled};
-        $grabbers->{$proggy}->{laststatus} = sprintf "enabled on %s, not run yet",(strftime "%a%d%b%y", localtime(time));
-    } elsif ($postprocessors->{$proggy}) {
-        delete $postprocessors->{$proggy}->{disabled};
-        $postprocessors->{$proggy}->{laststatus} = sprintf "enabled on %s, not run yet",(strftime "%a%d%b%y", localtime(time));
-    }
+    delete $components->{$proggy}->{disabled};
+    $components->{$proggy}->{laststatus} = sprintf "enabled on %s, not run yet",(strftime "%a%d%b%y", localtime(time));
     $made_changes = 1;
 }
@@ -1368 +1333 @@
 
     # confirm it exists first
-    if ((!$grabbers->{$proggy}) && (!$postprocessors->{$proggy})) {
-        printf "No such grabber/postprocessor: \"%s\".\n",$proggy;
+    if (!$components->{$proggy}) {
+        printf "No such component: \"%s\".\n",$proggy;
         return;
     }
     print "Disabling $proggy.\n";
 
-    if ($grabbers->{$proggy}) {
-        $grabbers->{$proggy}->{disabled} = 1;
-        $grabbers->{$proggy}->{laststatus} = sprintf "manually disabled on %s",(strftime "%a%d%b%y", localtime(time));
-    } elsif ($postprocessors->{$proggy}) {
-        $postprocessors->{$proggy}->{disabled} = 1;
-        $postprocessors->{$proggy}->{laststatus} = sprintf "manually disabled on %s",(strftime "%a%d%b%y", localtime(time));
-    }
+    $components->{$proggy}->{disabled} = 1;
+    $components->{$proggy}->{laststatus} = sprintf "manually disabled on %s",(strftime "%a%d%b%y", localtime(time));
     $made_changes = 1;
 }
@@ -1390 +1350 @@
 
     # reset current order to zero
-    foreach my $proggy (keys %$grabbers) {
-        $grabbers->{$proggy}->{order} = 0;
+    foreach my $proggy (query_grabbers()) {
+        $components->{$proggy}->{order} = 0;
     }
 
@@ -1398 +1358 @@
     if ($pref_order) {
         foreach my $proggy (split(/,/,$pref_order)) {
-            if (defined $grabbers->{$proggy}) {
-                $grabbers->{$proggy}->{order} = $order_num;
+            if (defined $components->{$proggy} and $components->{$proggy}->{type} eq 'grabber') {
+                $components->{$proggy}->{order} = $order_num;
                 $order_num++;
             }
@@ -1406 +1366 @@
 
     # set order of any grabbers not specified in a random manner
-    foreach my $proggy (sort keys %$grabbers) {
-        if ((!defined $grabbers->{$proggy}->{order}) || ($grabbers->{$proggy}->{order} == 0)) {
-            $grabbers->{$proggy}->{order} = $order_num+int(rand(1000));
+    foreach my $proggy (sort query_grabbers()) {
+        if ((!defined $components->{$proggy}->{order}) || ($components->{$proggy}->{order} == 0)) {
+            $components->{$proggy}->{order} = $order_num+int(rand(1000));
         }
     }
@@ -1415 +1375 @@
     print "Grabber order set as follows:\n" unless $quiet;
     $order_num = 0;
-    foreach my $proggy (sort { $grabbers->{$a}->{order} <=> $grabbers->{$b}->{order} } keys %$grabbers) {
+    foreach my $proggy (sort { $components->{$a}->{order} <=> $components->{$b}->{order} } query_grabbers()) {
         $order_num++;
-        $grabbers->{$proggy}->{order} = $order_num;
-        printf " #%d. %s%s\n",$grabbers->{$proggy}->{order},$proggy,($grabbers->{$proggy}->{disabled} ? " [disabled]" : "") unless $quiet;
+        $components->{$proggy}->{order} = $order_num;
+        printf " #%d. %s%s\n",$components->{$proggy}->{order},$proggy,($components->{$proggy}->{disabled} ? " [disabled]" : "") unless $quiet;
     }
 
@@ -1427 +1387 @@
 {
     my $result;
-    foreach my $proggy (keys %$grabbers) {
-        $result = test_proggy("$GRABBER_DIR/$proggy","$GRABBER_DIR/$proggy/$proggy");
-        printf "Grabber %s: %s\n",$proggy,($result ? "OK" : "Failed");
-        if (!$result ne !$grabbers->{$proggy}->{ready}) {
-            $grabbers->{$proggy}->{ready} = $result;
+    foreach my $proggy (keys %$components) {
+        my $progtype = $components->{$proggy}->{type};
+        $result = test_proggy("$CWD/$progtype" . "s/$proggy", "$CWD/$progtype" . "s/$proggy/$proggy");
+        printf "%s %s: %s\n",ucfirst($progtype), $proggy,($result ? "OK" : "Failed");
+        if (!$result ne !$components->{$proggy}->{ready}) {
+            $components->{$proggy}->{ready} = $result;
             $made_changes = 1;
         }
     }
-
-    foreach my $proggy (keys %$postprocessors) {
-        $result = test_proggy("$POSTPROCESSOR_DIR/$proggy","$POSTPROCESSOR_DIR/$proggy/$proggy");
-        printf "Postprocessor %s: %s\n",$proggy,($result ? "OK" : "Failed");
-        if (!$result ne !$postprocessors->{$proggy}->{ready}) {
-            $postprocessors->{$proggy}->{ready} = $result;
-            $made_changes = 1;
-        }
-    }
+}
+# -----------------------------------------
+# Subs: Utilities
+# -----------------------------------------
+#
+
+sub query_grabbers
+{
+    return query_component_type('grabber');
+}
+
+sub query_postprocessors
+{
+    return query_component_type('postprocessor');
+}
+
+sub query_component_type
+{
+    my $progtype = shift;
+
+    my @ret = ();
+    foreach (keys %$components)
+    {
+        push (@ret, $_) if ($components->{$_}->{type} eq $progtype);
+    }
+    return @ret;
 }
 
@@ -1454 +1432 @@
     read_file($config_file, 'configuration');
 
+    # TEMPORARY! Convert old $grabbers/$postprocessors config file to
+    # new $components format.
+    if (defined $grabbers or defined $postprocessors)
+    {
+        foreach (keys %$grabbers) {
+            $grabbers->{$_}->{type} = 'grabber';
+        }
+        foreach (keys %$postprocessors) {
+            $postprocessors->{$_}->{type} = 'postprocessor';
+        }
+        $components = { %$grabbers, %$postprocessors };
+        $grabbers = undef;
+        $postprocessors = undef;
+    }
+
+
     # if we are updating from a previous rev of shepherd.conf we may not
     # have any 'order' fields set .. check here
     my $found_order = 1;
-    foreach (keys %$grabbers)
-    {
-        $found_order = 0 if (!defined $grabbers->{$_}->{order});
+    foreach (query_grabbers())
+    {
+        $found_order = 0 if (!defined $components->{$_}->{order});
     }
     if (($found_order == 0) && (!$opt->{setorder}))
@@ -1511 +1505 @@
     open(CONF, ">$config_file") or die "cannot write to $config_file: $!";
     print CONF Data::Dumper->Dump(
-        [$region,  $pref_order,  $mirror_site,  $grabbers,  $postprocessors,  $title_translation_table  ],
-        ["region", "pref_order", "mirror_site", "grabbers", "postprocessors", "title_translation_table" ]);
+        [$region,  $pref_order,  $mirror_site,  $components, $title_translation_table  ],
+        ["region", "pref_order", "mirror_site", "components", "title_translation_table" ]);
     close CONF;
     print "\nUpdated configuration file $config_file.\n" if ($debug);
@@ -1706 +1700 @@
     print " Grabber           Version Enabled Ready Last Run   Status\n" .
           " ----------------- ------- ------- ----- ---------- ---------------------------\n";
-    foreach (sort { $grabbers->{$a}->{order} <=> $grabbers->{$b}->{order} } keys %$grabbers) {
-        my $h = $grabbers->{$_};
+    foreach (sort { $components->{$a}->{order} <=> $components->{$b}->{order} } query_grabbers()) {
+        my $h = $components->{$_};
         printf  " %-16s %8s %4s %6s  %11s %s\n",
                 "$h->{order}. $_",
@@ -1720 +1714 @@
     print " Postprocessor     Version Enabled Ready Last Run   Status\n" .
           " ----------------- ------- ------- ----- ---------- ---------------------------\n";
-    foreach (sort { $postprocessors->{$a} <=> $postprocessors->{$b} } keys %$postprocessors) {
-        my $h = $postprocessors->{$_};
+    foreach (sort { $components->{$a} <=> $components->{$b} } query_postprocessors()) {
+        my $h = $components->{$_};
         printf  " %-16s %8s %4s %6s  %11s %s\n",
                 $_,