Changeset 60

Timestamp:

10/10/06 17:10:10 (7 years ago)

Author:

lincoln

Message:

start of reconciler simplification: additional data checks and printed warnings

Files:

• shepherd (modified) (2 diffs)

shepherd

@@ -499 +499 @@
 
     my $parse_start_time = [gettimeofday];
+    my $this_plugin = $plugin_data->{$plugin};
     printf STDERR "SHEPHERD: Started parsing XMLTV from '%s' in '%s' .. any errors below are from parser:\n",$plugin,$output;
-    eval { $plugin_data->{$plugin}->{xmltv} = XMLTV::parsefiles($output); };
+    eval { $this_plugin->{xmltv} = XMLTV::parsefiles($output); };
     printf STDERR "SHEPHERD: Completed XMLTV parsing from '%s' in %0.2f seconds\n",$plugin,tv_interval($parse_start_time);
 
-    if (defined $plugin_data->{$plugin}->{xmltv}) {
-        $plugin_data->{$plugin}->{valid} = 1;
-
-        my $xmltv = $plugin_data->{$plugin}->{xmltv};
-        my ($encoding, $credits, $chan, $progs) = @$xmltv;
-        $plugin_data->{$plugin}->{total_duration} = 0;
-        $plugin_data->{$plugin}->{programmes} = 0;
-
-        my $strptime = new DateTime::Format::Strptime( pattern => "%Y%m%d%H%M %z");
-        my $alt_strptime = new DateTime::Format::Strptime( pattern => "%Y%m%d%H%M"); # alternate format 1: oztivo doesn't seem to output timezone
-        my $seen_channels_with_data = 0;
-
-        # iterate thru channels
-        foreach my $ch (sort keys %{$channels}) {
-            my $seen_progs_on_this_channel = 0;
-
-            # iterate thru programmes per channel
-            foreach my $prog (@$progs) {
-                next if ($prog->{channel} ne $channels->{$ch});
-
-                my $t1 = $strptime->parse_datetime($prog->{start});
-                $t1 = $alt_strptime->parse_datetime($prog->{start}) if (!$t1);
-
-                my $t2 = $strptime->parse_datetime($prog->{stop});
-                $t2 = $alt_strptime->parse_datetime($prog->{stop}) if (!$t2);
-
-                next if (!$t1 || !$t2); # if we can't parse stop/start then clearly THIS data is bunk!
-
-                # store t1 and t2 times in the xmltv data for later on (shh.. ton't tell anyone..)
-                $prog->{start_epoch} = $t1->epoch;
-                $prog->{stop_epoch} = $t2->epoch;
-
-                # store plugin-specific stats
-                $plugin_data->{$plugin}->{programmes}++;
-                $plugin_data->{$plugin}->{total_duration} += ($t2->epoch - $t1->epoch);
-                $seen_progs_on_this_channel++;
-                $plugin_data->{$plugin}->{earliest_data_seen} = $t1->epoch if (!defined $plugin_data->{$plugin}->{earliest_data_seen});
-                $plugin_data->{$plugin}->{earliest_data_seen} = $t1->epoch if ($t1->epoch < $plugin_data->{$plugin}->{earliest_data_seen});
-                $plugin_data->{$plugin}->{latest_data_seen} = $t2->epoch if (!defined $plugin_data->{$plugin}->{latest_data_seen});
-                $plugin_data->{$plugin}->{latest_data_seen} = $t2->epoch if ($t2->epoch > $plugin_data->{$plugin}->{latest_data_seen});
-
-                # store channel-specific stats
-                $channel_data->{$ch}->{programmes}++;
-                $channel_data->{$ch}->{total_duration} += ($t2->epoch - $t1->epoch);
-
-                # store timeslot info
-                next if ($t1->epoch > $endtime);        # programme starts after timeslots we are interested .. nice that we have it ... but we really don't care about it!
-                next if ($t2->epoch < $starttime);      # programme ends  before timeslots we are interested .. nice that we have it ... but we really don't care about it!
-                my $start_slotnum;
-                if ($t1->epoch >= $starttime) {
-                    $start_slotnum = int(($t1->epoch - $starttime) / $timeslot_size);
-                } else {
-                    $start_slotnum = 0;
-                }
-                my $end_slotnum;
-                if ($t2->epoch < $endtime) {
-                    $end_slotnum = int(($t2->epoch - $starttime) / $timeslot_size);
-                } else {
-                    $end_slotnum = ($num_timeslots-1);
-                }
-
-                # add this programme into the global timeslots table for this channel
-                foreach my $slotnum ($start_slotnum..$end_slotnum) {
-                    $channel_data->{$ch}->{timeslots}[$slotnum]++;
-                }
+    if (!($this_plugin->{xmltv})) {
+        printf "WARNING: Plugin %s didn't seem to return any valid XMLTV!\n",$plugin;
+        return;
+    }
+
+    $this_plugin->{valid} = 1;
+
+    my $xmltv = $this_plugin->{xmltv};
+    my ($encoding, $credits, $chan, $progs) = @$xmltv;
+    $this_plugin->{total_duration} = 0;
+    $this_plugin->{programmes} = 0;
+    $this_plugin->{progs_with_invalid_date} = 0;        # explicitly track unparsable dates
+    $this_plugin->{progs_with_unknown_channel} = 0;     # explicitly track unknown channels
+
+    my $strptime = new DateTime::Format::Strptime( pattern => "%Y%m%d%H%M %z");
+    my $alt_strptime = new DateTime::Format::Strptime( pattern => "%Y%m%d%H%M"); # alternate format 1: oztivo doesn't seem to output timezone
+    my $seen_channels_with_data = 0;
+
+    #
+    # first iterate through all programmes and see if there are any channels we don't know about
+    #
+    my %chan_xml_list;
+    foreach my $ch (sort keys %{$channels}) {
+        $chan_xml_list{($channels->{$ch})} = 1;
+    }
+    foreach my $prog (@$progs) {
+        if (!defined $chan_xml_list{($prog->{channel})}) {
+            $this_plugin->{progs_with_unknown_channel}++;
+            printf " - WARNING: plugin '%s' returned data for unknown channel '%s': ignored.\n",$plugin,$prog->{channel};
+            $chan_xml_list{($prog->{channel})} = 1;     # so we warn only once
+        }
+    }
+        
+    # iterate thru channels
+    foreach my $ch (sort keys %{$channels}) {
+        my $seen_progs_on_this_channel = 0;
+
+        # iterate thru programmes per channel
+        foreach my $prog (@$progs) {
+            next if ($prog->{channel} ne $channels->{$ch});
+
+            my $t1 = $strptime->parse_datetime($prog->{start});
+            my $t2 = $strptime->parse_datetime($prog->{stop});
+            $t1 = $alt_strptime->parse_datetime($prog->{start}) if (!$t1);
+            $t2 = $alt_strptime->parse_datetime($prog->{stop}) if (!$t2);
+
+            if (!$t1 || !$t2) {
+                printf " - WARNING: plugin '%s' returned programme data with invalid timestamp format: \"%s\": can't parse.\n",
+                    $plugin,(!$t1 ? $prog->{start} : $prog->{stop}) if (!$this_plugin->{progs_with_invalid_date});
+                $this_plugin->{progs_with_invalid_date}++;
+                next;
             }
 
-            $seen_channels_with_data++ if ($seen_progs_on_this_channel > 0);
-        }
-
-        # print some stats about what we saw!
-        printf "SHEPHERD: %s '%s' returned data for %d channels, %d programmes, %dd%02dh%02dm%02ds duration, %s%s\n",
-            ucfirst($plugintype), $plugin, $seen_channels_with_data, $plugin_data->{$plugin}->{programmes},
-            int($plugin_data->{$plugin}->{total_duration} / 86400),             # days
-            int(($plugin_data->{$plugin}->{total_duration} % 86400) / 3600),    # hours
-            int(($plugin_data->{$plugin}->{total_duration} % 3600) / 60),       # mins
-            int($plugin_data->{$plugin}->{total_duration} % 60),                # sec
-            (defined $plugin_data->{$plugin}->{earliest_data_seen} ? (strftime "%a %e %b %H:%M - ", localtime($plugin_data->{$plugin}->{earliest_data_seen})) : 'no data'),
-            (defined $plugin_data->{$plugin}->{latest_data_seen} ? (strftime "%a %e %b %H:%M", localtime($plugin_data->{$plugin}->{latest_data_seen})) : '');
-        $plugin_data->{$plugin}->{laststatus} = sprintf "%dch/%dpr/%dhrs %s-%s",
-            $seen_channels_with_data, $plugin_data->{$plugin}->{programmes},
-            int($plugin_data->{$plugin}->{total_duration} / 3600),
-            (defined $plugin_data->{$plugin}->{earliest_data_seen} ? (strftime "%a%d%b%H:%M", localtime($plugin_data->{$plugin}->{earliest_data_seen})) : 'no'),
-            (defined $plugin_data->{$plugin}->{latest_data_seen} ? (strftime "%a%d%b%H:%M", localtime($plugin_data->{$plugin}->{latest_data_seen})) : 'data');
-
-    } else {
-        printf "WARNING: Plugin %s didn't seem to return any valid XMLTV!\n",$plugin;
-        delete $plugin_data->{$plugin}->{valid};
-    }
+            # store t1 and t2 times in the xmltv data for later on (shh.. ton't tell anyone..)
+            $prog->{start_epoch} = $t1->epoch;
+            $prog->{stop_epoch} = $t2->epoch;
+
+            # store plugin-specific stats
+            $this_plugin->{programmes}++;
+            $this_plugin->{total_duration} += ($t2->epoch - $t1->epoch);
+            $seen_progs_on_this_channel++;
+            $this_plugin->{earliest_data_seen} = $t1->epoch if (!defined $this_plugin->{earliest_data_seen});
+            $this_plugin->{earliest_data_seen} = $t1->epoch if ($t1->epoch < $this_plugin->{earliest_data_seen});
+            $this_plugin->{latest_data_seen} = $t2->epoch if (!defined $this_plugin->{latest_data_seen});
+            $this_plugin->{latest_data_seen} = $t2->epoch if ($t2->epoch > $this_plugin->{latest_data_seen});
+
+            # store channel-specific stats
+            $channel_data->{$ch}->{programmes}++;
+            $channel_data->{$ch}->{total_duration} += ($t2->epoch - $t1->epoch);
+
+            # store timeslot info
+            next if ($t1->epoch > $endtime);    # programme starts after timeslots we are interested .. nice that we have it ... but we really don't care about it!
+            next if ($t2->epoch < $starttime);  # programme ends  before timeslots we are interested .. nice that we have it ... but we really don't care about it!
+            my $start_slotnum;
+            if ($t1->epoch >= $starttime) {
+                $start_slotnum = int(($t1->epoch - $starttime) / $timeslot_size);
+            } else {
+                $start_slotnum = 0;
+            }
+            my $end_slotnum;
+            if ($t2->epoch < $endtime) {
+                $end_slotnum = int(($t2->epoch - $starttime) / $timeslot_size);
+            } else {
+                $end_slotnum = ($num_timeslots-1);
+            }
+
+            # add this programme into the global timeslots table for this channel
+            foreach my $slotnum ($start_slotnum..$end_slotnum) {
+                $channel_data->{$ch}->{timeslots}[$slotnum]++;
+            }
+        }
+
+        $seen_channels_with_data++ if ($seen_progs_on_this_channel > 0);
+    }
+
+    # print some stats about what we saw!
+    printf "SHEPHERD: %s '%s' returned data for %d channels, %d programmes, %dd%02dh%02dm%02ds duration, %s%s\n",
+        ucfirst($plugintype), $plugin, $seen_channels_with_data, $this_plugin->{programmes},
+        int($this_plugin->{total_duration} / 86400),            # days
+        int(($this_plugin->{total_duration} % 86400) / 3600),   # hours
+        int(($this_plugin->{total_duration} % 3600) / 60),      # mins
+        int($this_plugin->{total_duration} % 60),               # sec
+        (defined $this_plugin->{earliest_data_seen} ? (strftime "%a %e %b %H:%M - ", localtime($this_plugin->{earliest_data_seen})) : 'no data'),
+        (defined $this_plugin->{latest_data_seen} ? (strftime "%a %e %b %H:%M", localtime($this_plugin->{latest_data_seen})) : '');
+    $this_plugin->{laststatus} = sprintf "%dch/%dpr/%dhrs %s-%s",
+        $seen_channels_with_data, $this_plugin->{programmes},
+        int($this_plugin->{total_duration} / 3600),
+        (defined $this_plugin->{earliest_data_seen} ? (strftime "%a%d%b%H:%M", localtime($this_plugin->{earliest_data_seen})) : 'no'),
+        (defined $this_plugin->{latest_data_seen} ? (strftime "%a%d%b%H:%M", localtime($this_plugin->{latest_data_seen})) : 'data');
+
+    $plugin_data->{$plugin} = $this_plugin;
 }
 
@@ -1006 +1029 @@
 # sorting helper routine - sort by channel then by start-time
 sub order_channel_time {
-        my $chanresult = $a->{channel} cmp $b->{channel};
-        return $chanresult if ($chanresult != 0);
-        return ($a->{start_epoch} <=> $b->{start_epoch});
+    my $chanresult = $a->{channel} cmp $b->{channel};
+    return $chanresult if ($chanresult != 0);
+    if (defined $a->{start_epoch} && defined $b->{start_epoch}) {
+        return ($a->{start_epoch} <=> $b->{start_epoch});
+    } else {
+        return 0;
+    }
 }