Changeset 57

Timestamp:

10/09/06 20:36:52 (7 years ago)

Author:

lincoln

Message:

Bugfix: don't call postprocessors that aren't ready, rework accept-data-or-not postprocessor logic

Files:

• shepherd (modified) (11 diffs)
• status (modified) (1 diff)

shepherd

@@ -3 +3 @@
 # "Shepherd"
 
-my $version = '0.2.9';
+my $version = '0.2.10';
 
 # A wrapper for various Aussie TV guide data grabbers
@@ -24 +24 @@
 # 0.2.8   : Integrated reconciler
 # 0.2.9   : Grabber config support
+# 0.2.10  : Bugfix: don't call postprocessors that aren't ready,
+#           rework accept-data-or-not postprocessor logic
 
 BEGIN { *CORE::GLOBAL::die = \&my_die; }
@@ -127 +129 @@
 my $starttime, my $endtime;
 my $input_postprocess_file = "$CWD/reconciled_output_pre_postprocessors.xml";
-my $insufficient_grabber_data = 0;
+my $grabber_data_percent = 0;
 
 # OBSOLETE: will be removed
@@ -245 +247 @@
 {
     my $used_grabbers = 0;
-    my $need_more_data = 1;
+    my $found_data_percent = 0;
 
     print "\nGrabber stage:\n";
@@ -289 +291 @@
 
         # check to see if we have all the data we want
-        $need_more_data = &analyze_plugin_data($channel_ok_threshold_percent, "AGGREGATE GRABBER");
-
-        last if (!$need_more_data);
+        $found_data_percent = &analyze_plugin_data($channel_ok_threshold_percent, "AGGREGATE GRABBER");
+
+        last if ($found_data_percent >= $channel_ok_threshold_percent);
     }
 
@@ -301 +303 @@
     }
 
-    if ($need_more_data)
+    if ($found_data_percent < $channel_ok_threshold_percent)
     {
         print "SHEPHERD: Ran through ALL grabbers but still missing data!!! :(\n";
-        $insufficient_grabber_data = 1;
+        $grabber_data_percent = $found_data_percent;
         return;
     }
@@ -594 +596 @@
 
 # analyze grabber data - do we have all the data we want?
-# returns 1 if we need more data, 0 if we have all we want
+# returns percent of data found
 sub analyze_plugin_data
 {
     my ($threshold,$analysistype) = @_;
-    my $retval = 0; # until proven otherwise
     my $total_data_percent = 0, my $total_channels = 0;
     my $statusstring = "";
@@ -617 +618 @@
             } else {
                 $statusstring .= sprintf "%s: %0.1f%% [hungry], ",$ch,$data_in_channel_percent;
-                $retval = 1;
             }
             $total_data_percent += $data_in_channel_percent;
         } else {
             $statusstring .= sprintf "%s: 0%% [starving], ",$ch;
-            $retval = 1;
         }
     }
@@ -636 +635 @@
         uc($analysistype), $statusstring, $total_data_percent,
         ($total_data_percent >= $channel_ok_threshold_percent ? ">" : "<"), $channel_ok_threshold_percent,
-        ($retval ? "WANT MORE DATA" : "COMPLETE");
-    return $retval;
+        (($total_data_percent < $channel_ok_threshold_percent) ? "WANT MORE DATA" : "COMPLETE");
+    return $total_data_percent;
 }
 
@@ -1043 +1042 @@
     # first time around: feed in reconciled data ($input_postprocess_file)
 
-    my $need_more_data;
-
     printf "\nPostprocessing stage:\n";
 
     foreach my $postprocessor (sort { $components->{$a} <=> $components->{$b} } query_postprocessors()) {
         next if ($components->{$postprocessor}->{disabled});
+        next if (!$components->{$postprocessor}->{ready});
 
         $components->{$postprocessor}->{lastdata} = time;
@@ -1087 +1085 @@
         # process and analyze it!
         &soak_up_data($postprocessor, $output, "postprocessor");
-        $need_more_data = &analyze_plugin_data($postprocessor_ok_threshold_percent, "POSTPROCESSOR");
+        my $found_data_percent = &analyze_plugin_data($postprocessor_ok_threshold_percent, "POSTPROCESSOR");
 
         $components->{$postprocessor}->{laststatus} = $plugin_data->{$postprocessor}->{laststatus};
 
-        if (($need_more_data) && (!$insufficient_grabber_data)) {
-            # urgh.  this postprocessor did a bad bad thing ...
-            printf "SHEPHERD: XML data from postprocessor %s rejected, using XML from previous stage\n",$postprocessor;
-
-            if (defined $components->{$postprocessor}->{conescutive_failures}) {
-                $components->{$postprocessor}->{conescutive_failures}++;
+        if ($found_data_percent < $postprocessor_ok_threshold_percent) {
+            # how bad is the data?  is it significantly different to that of what the grabber run finished with?
+            # allow at most 5% of the data to go away
+            if ($grabber_data_percent > ($found_data_percent + 5)) {
+                # urgh.  this postprocessor did a bad bad thing ...
+                printf "SHEPHERD: XML data from postprocessor %s rejected, using XML from previous stage\n",$postprocessor;
+
+                if (defined $components->{$postprocessor}->{conescutive_failures}) {
+                    $components->{$postprocessor}->{conescutive_failures}++;
+                } else {
+                    $components->{$postprocessor}->{conescutive_failures} = 1;
+                }
+                printf "SHEPHERD: Postprocessor \"%s\" has now failed %d times in a row.  %d more and it will be automatically disabled.\n",
+                    $postprocessor,
+                    $components->{$postprocessor}->{conescutive_failures},
+                    ($postprocessor_disable_failure_threshold - $components->{$postprocessor}->{conescutive_failures});
+
+                if ($components->{$postprocessor}->{conescutive_failures} >= $postprocessor_disable_failure_threshold) {
+                    printf "SHEPHERD: Disabling Postprocessor \"%s\".\n",$postprocessor;
+                    $components->{$postprocessor}->{disabled} = 1;
+                }
             } else {
-                $components->{$postprocessor}->{conescutive_failures} = 1;
-            }
-            printf "SHEPHERD: Postprocessor \"%s\" has now failed %d times in a row.  %d more and it will be automatically disabled.\n",
-                $postprocessor,
-                $components->{$postprocessor}->{conescutive_failures},
-                ($postprocessor_disable_failure_threshold - $components->{$postprocessor}->{conescutive_failures});
-
-            if ($components->{$postprocessor}->{conescutive_failures} >= $postprocessor_disable_failure_threshold) {
-                printf "SHEPHERD: Disabling Postprocessor \"%s\".\n",$postprocessor;
-                $components->{$postprocessor}->{disabled} = 1;
+                # accept what this postprocessor did to our output ...
+                printf "SHEPHERD: accepting output from postprocessor %s, feeding it into next stage\n",$postprocessor;
+                $input_postprocess_file = $output;
+                delete $components->{$postprocessor}->{conescutive_failures} if (defined $components->{$postprocessor}->{conescutive_failures});
             }
         } else {

status

@@ -1 @@
-shepherd:0.2.9:shepherd
+shepherd:0.2.10:shepherd
 rex:3.2.1-r2:grabber
 oztivo:0.4-r2:grabber