Changeset 10

Timestamp:

10/04/06 15:32:02 (7 years ago)

Author:

lincoln

Message:

some code refactoring, cutdown writes to config_file if nothing changed, first pass at making XMLTV parsing more bulletproof (not there yet)

Files:

• shepherd (modified) (16 diffs)

shepherd

@@ -40 +40 @@
 use DateTime::Format::Strptime;
 use POSIX qw(strftime);
+use Time::HiRes qw(gettimeofday tv_interval);
 
 # ---------------------------------------------------------------------------
@@ -152 +153 @@
     update($progname, $version);
     set_order(1) if $made_changes;
-    write_config_file() unless ($opt->{configure});
+    write_config_file() if (($made_changes) && (! $opt->{configure}))
 }
 
@@ -190 +191 @@
     my $need_more_data = 1;
 
+    printf "\nGrabber stage:\n";
+
     # iterate across grabbers until we have all our data we want (or need)
     foreach my $grabber (sort { $grabbers->{$a}->{order} <=> $grabbers->{$b}->{order} } keys %$grabbers) {
@@ -198 +201 @@
         $grabbers->{$grabber}->{laststatus} = "unknown";
 
-        printf "SHEPHERD: Using grabber: (%d) %s\n",$grabbers->{$grabber}->{order},$grabber;
+        printf "\nSHEPHERD: Using grabber: (%d) %s\n",$grabbers->{$grabber}->{order},$grabber;
 
         my $output = "$GRABBER_DIR/$grabber/output.xmltv";
@@ -222 +225 @@
 
         # soak up the data we just collected
-        &soak_up_data($grabber, $output);
+        &soak_up_data($grabber, $output, "grabber");
         $grabbers->{$grabber}->{laststatus} = $plugin_data->{$grabber}->{laststatus};
 
         # check to see if we have all the data we want
-        $need_more_data = &analyze_plugin_data($channel_ok_threshold_percent);
+        $need_more_data = &analyze_plugin_data($channel_ok_threshold_percent, "AGGREGATE GRABBER");
 
         last if (!$need_more_data);
@@ -250 +253 @@
 sub soak_up_data
 {
-    my ($grabber, $output) = @_;
-    eval { $plugin_data->{$grabber}->{xmltv} = XMLTV::parsefiles($output); };
-
-    if (defined $plugin_data->{$grabber}->{xmltv}) {
-        $plugin_data->{$grabber}->{valid} = 1;
-
-        my $xmltv = $plugin_data->{$grabber}->{xmltv};
+    my ($plugin, $output, $plugintype) = @_;
+
+    if (! -r $output) {
+        printf "SHEPHERD: Warning: plugin '%s' output file '%s' does not exist\n",$plugin,$output;
+        return;
+    }
+
+    my $parse_start_time = [gettimeofday];
+    printf STDERR "SHEPHERD: Started parsing XMLTV from '%s' in '%s' .. any errors below are from parser:\n",$plugin,$output;
+    eval {
+        local $SIG{__DIE__}, 
+        $plugin_data->{$plugin}->{xmltv} = XMLTV::parsefiles($output);
+    };
+    printf STDERR "SHEPHERD: Warning: Parser returned the following: $@\n" if $@;
+    printf STDERR "SHEPHERD: Completed XMLTV parsing from '%s' in %0.2f seconds\n",$plugin,tv_interval($parse_start_time);
+
+    if (defined $plugin_data->{$plugin}->{xmltv}) {
+        $plugin_data->{$plugin}->{valid} = 1;
+
+        my $xmltv = $plugin_data->{$plugin}->{xmltv};
         my ($encoding, $credits, $chan, $progs) = @$xmltv;
-        $plugin_data->{$grabber}->{total_duration} = 0;
-        $plugin_data->{$grabber}->{programmes} = 0;
+        $plugin_data->{$plugin}->{total_duration} = 0;
+        $plugin_data->{$plugin}->{programmes} = 0;
 
         my $strptime = new DateTime::Format::Strptime( pattern => "%Y%m%d%H%M %z");
@@ -276 +292 @@
                 next if (!$t1 || !$t2); # if we can't parse stop/start then clearly THIS data is bunk!
 
-                # store grabber-specific stats
-                $plugin_data->{$grabber}->{programmes}++;
-                $plugin_data->{$grabber}->{total_duration} += ($t2->epoch - $t1->epoch);
+                # store plugin-specific stats
+                $plugin_data->{$plugin}->{programmes}++;
+                $plugin_data->{$plugin}->{total_duration} += ($t2->epoch - $t1->epoch);
                 $seen_progs_on_this_channel++;
-                $plugin_data->{$grabber}->{earliest_data_seen} = $t1->epoch if (!defined $plugin_data->{$grabber}->{earliest_data_seen});
-                $plugin_data->{$grabber}->{earliest_data_seen} = $t1->epoch if ($t1->epoch < $plugin_data->{$grabber}->{earliest_data_seen});
-                $plugin_data->{$grabber}->{latest_data_seen} = $t2->epoch if (!defined $plugin_data->{$grabber}->{latest_data_seen});
-                $plugin_data->{$grabber}->{latest_data_seen} = $t2->epoch if ($t2->epoch > $plugin_data->{$grabber}->{latest_data_seen});
+                $plugin_data->{$plugin}->{earliest_data_seen} = $t1->epoch if (!defined $plugin_data->{$plugin}->{earliest_data_seen});
+                $plugin_data->{$plugin}->{earliest_data_seen} = $t1->epoch if ($t1->epoch < $plugin_data->{$plugin}->{earliest_data_seen});
+                $plugin_data->{$plugin}->{latest_data_seen} = $t2->epoch if (!defined $plugin_data->{$plugin}->{latest_data_seen});
+                $plugin_data->{$plugin}->{latest_data_seen} = $t2->epoch if ($t2->epoch > $plugin_data->{$plugin}->{latest_data_seen});
 
                 # store channel-specific stats
@@ -315 +331 @@
 
         # print some stats about what we saw!
-        printf "SHEPHERD: Grabber '%s' returned data for %d channels, %d programmes, %dd%02dh%02dm%02ds duration, %s%s\n",
-            $grabber, $seen_channels_with_data, $plugin_data->{$grabber}->{programmes},
-            int($plugin_data->{$grabber}->{total_duration} / 86400),            # days
-            int(($plugin_data->{$grabber}->{total_duration} % 86400) / 3600),   # hours
-            int(($plugin_data->{$grabber}->{total_duration} % 3600) / 60),      # mins
-            int($plugin_data->{$grabber}->{total_duration} % 60),               # sec
-            (defined $plugin_data->{$grabber}->{earliest_data_seen} ? (strftime "%a %e %b %H:%M - ", localtime($plugin_data->{$grabber}->{earliest_data_seen})) : 'no data'),
-            (defined $plugin_data->{$grabber}->{latest_data_seen} ? (strftime "%a %e %b %H:%M", localtime($plugin_data->{$grabber}->{latest_data_seen})) : '');
-        $plugin_data->{$grabber}->{laststatus} = sprintf "%dch/%dpr/%dhrs %s-%s",
-            $seen_channels_with_data, $plugin_data->{$grabber}->{programmes},
-            int($plugin_data->{$grabber}->{total_duration} / 3600),
-            (defined $plugin_data->{$grabber}->{earliest_data_seen} ? (strftime "%a%d%b%H:%M", localtime($plugin_data->{$grabber}->{earliest_data_seen})) : 'no'),
-            (defined $plugin_data->{$grabber}->{latest_data_seen} ? (strftime "%a%d%b%H:%M", localtime($plugin_data->{$grabber}->{latest_data_seen})) : 'data');
+        printf "SHEPHERD: %s '%s' returned data for %d channels, %d programmes, %dd%02dh%02dm%02ds duration, %s%s\n",
+            ucfirst($plugintype), $plugin, $seen_channels_with_data, $plugin_data->{$plugin}->{programmes},
+            int($plugin_data->{$plugin}->{total_duration} / 86400),             # days
+            int(($plugin_data->{$plugin}->{total_duration} % 86400) / 3600),    # hours
+            int(($plugin_data->{$plugin}->{total_duration} % 3600) / 60),       # mins
+            int($plugin_data->{$plugin}->{total_duration} % 60),                # sec
+            (defined $plugin_data->{$plugin}->{earliest_data_seen} ? (strftime "%a %e %b %H:%M - ", localtime($plugin_data->{$plugin}->{earliest_data_seen})) : 'no data'),
+            (defined $plugin_data->{$plugin}->{latest_data_seen} ? (strftime "%a %e %b %H:%M", localtime($plugin_data->{$plugin}->{latest_data_seen})) : '');
+        $plugin_data->{$plugin}->{laststatus} = sprintf "%dch/%dpr/%dhrs %s-%s",
+            $seen_channels_with_data, $plugin_data->{$plugin}->{programmes},
+            int($plugin_data->{$plugin}->{total_duration} / 3600),
+            (defined $plugin_data->{$plugin}->{earliest_data_seen} ? (strftime "%a%d%b%H:%M", localtime($plugin_data->{$plugin}->{earliest_data_seen})) : 'no'),
+            (defined $plugin_data->{$plugin}->{latest_data_seen} ? (strftime "%a%d%b%H:%M", localtime($plugin_data->{$plugin}->{latest_data_seen})) : 'data');
 
     } else {
-        printf "WARNING: Plugin %s didn't seem to return any valid XMLTV!\n",$grabber;
-        delete $plugin_data->{$grabber}->{valid};
+        printf "WARNING: Plugin %s didn't seem to return any valid XMLTV!\n",$plugin;
+        delete $plugin_data->{$plugin}->{valid};
     }
 }
@@ -340 +356 @@
 sub analyze_plugin_data
 {
-    my $threshold = shift;
+    my ($threshold,$analysistype) = @_;
     my $retval = 0; # until proven otherwise
     my $total_data_percent = 0, my $total_channels = 0;
@@ -376 +392 @@
 
     # print some stats about what our analysis says!
-    printf "SHEPHERD: ANALYSIS: %sTOTAL %0.2f%% %s %0.2f%%: %s\n", $statusstring, $total_data_percent,
+    printf "SHEPHERD: %s ANALYSIS: %sTOTAL %0.2f%% %s %0.2f%%: %s\n",
+        uc($analysistype), $statusstring, $total_data_percent,
         ($total_data_percent >= $channel_ok_threshold_percent ? ">" : "<"), $channel_ok_threshold_percent,
         ($retval ? "WANT MORE DATA" : "COMPLETE");
@@ -416 +433 @@
     my $need_more_data;
 
+    printf "\nPostprocessing stage:\n";
+
     foreach my $postprocessor (sort { $postprocessors->{$a} <=> $postprocessors->{$b} } keys %$postprocessors) {
         next if ($postprocessors->{$postprocessor}->{disabled});
@@ -422 +441 @@
         $postprocessors->{$postprocessor}->{laststatus} = "unknown";
 
-        printf "SHEPHERD: Using postprocessor: %s\n",$postprocessor;
+        printf "\nSHEPHERD: Using postprocessor: %s\n",$postprocessor;
 
         my $output = "$POSTPROCESSOR_DIR/$postprocessor/output.xmltv";
@@ -455 +474 @@
 
         # process and analyze it!
-        &soak_up_data($postprocessor, $output);
-        $need_more_data = &analyze_plugin_data($postprocessor_ok_threshold_percent);
+        &soak_up_data($postprocessor, $output, "postprocessor");
+        $need_more_data = &analyze_plugin_data($postprocessor_ok_threshold_percent, "POSTPROCESSOR");
 
         $postprocessors->{$postprocessor}->{laststatus} = $plugin_data->{$postprocessor}->{laststatus};
@@ -472 +491 @@
                 $postprocessor,
                 $postprocessors->{$postprocessor}->{conescutive_failures},
-                $postprocessor_disable_failure_threshold;
+                ($postprocessor_disable_failure_threshold - $postprocessors->{$postprocessor}->{conescutive_failures});
 
             if ($postprocessors->{$postprocessor}->{conescutive_failures} >= $postprocessor_disable_failure_threshold) {
@@ -521 +540 @@
 sub update
 {
-    print "\nFetching status file: $STATUS.\n";
+    printf "\nChecking for updates:\n\n";
+
+    print "Fetching status file: $STATUS.";
     my $data = LWP::Simple::get($STATUS);
     unless ($data)
@@ -561 +582 @@
 {
     my ($proggy, $latestversion, $progtype) = @_;
-
-    print "\n";
 
     # handle new installs..