Changeset 4

Timestamp:

10/03/06 23:58:47 (7 years ago)

Author:

max

Message:

Lincoln's updated XMLTV analysis.

Files:

• grabbers/abc2_website (modified) (3 diffs)
• grabbers/yahoo7widget (modified) (5 diffs)
• shepherd (modified) (10 diffs)
• status (modified) (1 diff)

grabbers/abc2_website

@@ -16 +16 @@
 
 my $progname = "abc2_website";
-my $version = "1.52_03oct06";
+my $version = "1.53_03oct06";
 
 use LWP::UserAgent;
@@ -422 +422 @@
 {
         my ($entry) = @_;
-        printf STDERR "[%d] %s\n",time,$entry;
+        printf STDERR "%s [%d] %s\n",$progname,time,$entry;
 }
 
@@ -429 +429 @@
 sub print_stats
 {
-        printf STDERR "completed in %0.2f seconds",tv_interval($script_start_time);
+        printf STDERR "%s v%s [%d] completed in %0.2f seconds",$progname,$version,time,tv_interval($script_start_time);
         foreach my $key (sort keys %stats) {
                 printf STDERR ", %d %s",$stats{$key},$key;

grabbers/yahoo7widget

@@ -16 +16 @@
 
 my $progname = "yahoo7widget";
-my $version = "1.52_03oct06";
+my $version = "1.53_03oct06";
 
 use LWP::UserAgent;
@@ -134 +134 @@
         &parse_xml_data( get_url(
                 (sprintf "http://au.tv.yahoo.com/widget.html?rg=%d&st=%d&et=%d", $region, $currtime,($currtime+86400)),
-                (sprintf "yahoo7widget data: day %d of %d",((($currtime-$starttime)/86400)+1),(($endtime-$starttime)/86400))));
+                (sprintf "yahoo7widget detailed data: day %d of %d",((($currtime-$starttime)/86400)+1),(($endtime-$starttime)/86400))));
 }
 
@@ -238 +238 @@
                 $request->header('X-Forwarded-For' => $randomaddr);
         }
-        &log(sprintf "[%d] fetching %s%s: %s\n",time,$status,($opt_obfuscate ? "[obfuscate]" : ""),$url);
+        &log(sprintf "fetching %s%s: %s",$status,($opt_obfuscate ? "[obfuscate]" : ""),$url);
         for (1..3) {
                 $response = $ua->request($request);
@@ -285 +285 @@
 {
         my ($entry) = @_;
-        printf STDERR "[%d] %s\n",time,$entry;
+        printf STDERR "%s [%d] %s\n",$progname,time,$entry;
 }
 
@@ -292 +292 @@
 sub print_stats
 {
-        printf STDERR "completed in %0.2f seconds",tv_interval($script_start_time);
+        printf STDERR "%s v%s [%d] completed in %0.2f seconds",$progname,$version,time,tv_interval($script_start_time);
         foreach my $key (sort keys %stats) {
                 printf STDERR ", %d %s",$stats{$key},$key;

shepherd

@@ -38 +38 @@
 use XMLTV;
 use XMLTV::Ask;
+use DateTime::Format::Strptime;
 
 # ---------------------------------------------------------------------------
@@ -58 +59 @@
 my $GRABBER_DIR = "$CWD/grabbers";
 my $ARCHIVE_DIR = "$CWD/archive";
+my $timeslot_size = (15 * 60); # 15 minute slots
+my $channel_ok_threshold_percent = 90;
 
 my $opt;
@@ -68 +71 @@
 my $channels_file = "$CWD/channels.conf";
 my $days;
+
+# postprocessing
+my $langs = [ 'en' ];
+my $num_timeslots;
 my $grabber_data = { };
-my $master_starttime = time;
+my $channel_data = { };
+my $starttime, my $endtime;
 
 # ---------------------------------------------------------------------------
@@ -150 +158 @@
 unless ($opt->{update})
 {
+    calc_date_range();
     grab_data();
 }
@@ -175 +184 @@
         $used_grabbers++;
 
-        print "\nUsing grabber: $grabber.\n\n";
-
-        my $output = $opt->{output};
-        $output ||= "$CWD/output.xmltv";
+        $grabber_data->{$grabber}->{last_grabbed} = time;
+
+        printf "SHEPHERD: Using grabber: (%d) %s\n",$grabbers->{$grabber}->{order},$grabber;
+
+        my $output = "$GRABBER_DIR/$grabber/output.xmltv";
         my $comm = "$GRABBER_DIR/$grabber/$grabber " .
                    "--region $region " .
@@ -191 +201 @@
         $comm .= " --debug" if ($debug);
         $comm .= " @ARGV" if (@ARGV);
-        print "Excuting command:\n$comm\n\n";
+        print "SHEPHERD: Excuting command: $comm\n";
 
         chdir "$GRABBER_DIR/$grabber/";
@@ -215 +225 @@
     if ($need_more_data)
     {
-        print "Ran through all grabbers but still missing data!!! :(\n";
+        print "SHEPHERD: Ran through ALL grabbers but still missing data!!! :(\n";
         return;
     }
@@ -225 +235 @@
 {
     my ($grabber, $output) = @_;
-    eval { $grabber_data->{$grabber} = XMLTV::parsefiles($output); };
+    eval { $grabber_data->{$grabber}->{xmltv} = XMLTV::parsefiles($output); };
+
+    if (defined $grabber_data->{$grabber}->{xmltv}) {
+        $grabber_data->{$grabber}->{valid} = 1;
+
+        my $xmltv = $grabber_data->{$grabber}->{xmltv};
+        my ($encoding, $credits, $chan, $progs) = @$xmltv;
+        $grabber_data->{$grabber}->{total_duration} = 0;
+        $grabber_data->{$grabber}->{programmes} = 0;
+
+        my $strptime = new DateTime::Format::Strptime( pattern => "%Y%m%d%H%M %z");
+        my $seen_channels_with_data = 0;
+
+        # iterate thru channels
+        foreach my $ch (sort keys %{$channels}) {
+            my $seen_progs_on_this_channel = 0;
+
+            # iterate thru programmes per channel
+            foreach my $prog (@$progs) {
+                next if ($prog->{channel} ne $channels->{$ch});
+
+                my $t1 = $strptime->parse_datetime($prog->{start});
+                my $t2 = $strptime->parse_datetime($prog->{stop});
+                next if (!$t1 || !$t2); # if we can't parse stop/start then clearly THIS data is bunk!
+
+                # store grabber-specific stats
+                $grabber_data->{$grabber}->{programmes}++;
+                $grabber_data->{$grabber}->{total_duration} += ($t2->epoch - $t1->epoch);
+                $seen_progs_on_this_channel++;
+                $grabber_data->{$grabber}->{earliest_data_seen} = $t1->epoch if (!defined $grabber_data->{$grabber}->{earliest_data_seen});
+                $grabber_data->{$grabber}->{earliest_data_seen} = $t1->epoch if ($t1->epoch < $grabber_data->{$grabber}->{earliest_data_seen});
+                $grabber_data->{$grabber}->{latest_data_seen} = $t2->epoch if (!defined $grabber_data->{$grabber}->{latest_data_seen});
+                $grabber_data->{$grabber}->{latest_data_seen} = $t2->epoch if ($t2->epoch > $grabber_data->{$grabber}->{latest_data_seen});
+
+                # store channel-specific stats
+                $channel_data->{$ch}->{programmes}++;
+                $channel_data->{$ch}->{total_duration} += ($t2->epoch - $t1->epoch);
+
+                # store timeslot info
+                next if ($t1->epoch > $endtime);        # programme starts after timeslots we are interested .. nice that we have it ... but we really don't care about it!
+                next if ($t2->epoch < $starttime);      # programme ends  before timeslots we are interested .. nice that we have it ... but we really don't care about it!
+                my $start_slotnum;
+                if ($t1->epoch >= $starttime) {
+                    $start_slotnum = int(($t1->epoch - $starttime) / $timeslot_size);
+                } else {
+                    $start_slotnum = 0;
+                }
+                my $end_slotnum;
+                if ($t2->epoch < $endtime) {
+                    $end_slotnum = int(($t2->epoch - $starttime) / $timeslot_size);
+                } else {
+                    $end_slotnum = ($num_timeslots-1);
+                }
+
+                # add this programme into the global timeslots table for this channel
+                foreach my $slotnum ($start_slotnum..$end_slotnum) {
+                    $channel_data->{$ch}->{timeslots}[$slotnum]++;
+                }
+            }
+
+            $seen_channels_with_data++ if ($seen_progs_on_this_channel > 0);
+        }
+
+        # print some stats about what we saw!
+        my $earlist_data_seen = localtime($grabber_data->{$grabber}->{earliest_data_seen});
+        my $latest_data_seen = localtime($grabber_data->{$grabber}->{latest_data_seen});
+        printf "SHEPHERD: Grabber '%s' returned data for %d channels (%d programmes, %dd%02dh%02dm%02ds duration, earliest %s, latest %s)\n",
+            $grabber, $seen_channels_with_data, $grabber_data->{$grabber}->{programmes},
+            int($grabber_data->{$grabber}->{total_duration} / 86400),           # days
+            int(($grabber_data->{$grabber}->{total_duration} % 86400) / 3600),  # hours
+            int(($grabber_data->{$grabber}->{total_duration} % 3600) / 60),     # mins
+            int($grabber_data->{$grabber}->{total_duration} % 60),              # sec
+            $earlist_data_seen, $latest_data_seen;
+    } else {
+        printf "WARNING: Grabber %s didn't seem to return any valid XMLTV!\n",$grabber;
+        delete $grabber_data->{$grabber}->{valid};
+    }
 }
 
@@ -233 +319 @@
 sub analyze_grabber_data
 {
+    my $retval = 0; # until proven otherwise
+    my $total_data_percent = 0, my $total_channels = 0;
+    my $statusstring = "";
+
+    # iterate across each channel
+    foreach my $ch (sort keys %{$channels}) {
+        $total_channels++;
+        if (defined $channel_data->{$ch}) {
+            my $data_in_channel = 0;
+            for my $slotnum (0..($num_timeslots-1)) {
+                $data_in_channel++ if ($channel_data->{$ch}->{timeslots}[$slotnum] > 0);
+            }
+
+            # do we have enough data for this channel?
+            my $data_in_channel_percent = $data_in_channel / ($num_timeslots-1) * 100;
+            if ($data_in_channel_percent >= $channel_ok_threshold_percent) {
+                $statusstring .= sprintf "%s: %0.1f%% [complete], ",$ch,$data_in_channel_percent;
+            } else {
+                $statusstring .= sprintf "%s: %0.1f%% [hungry], ",$ch,$data_in_channel_percent;
+                $retval = 1;
+            }
+            $total_data_percent += $data_in_channel_percent;
+        } else {
+            $statusstring .= sprintf "%s: 0%% [starving], ",$ch;
+            $retval = 1;
+        }
+    }
+
+    if ($total_channels > 0) {
+        $total_data_percent = $total_data_percent / $total_channels;
+    } else {
+        $total_data_percent = 0;
+    }
+
+    # print some stats about what our analysis says!
+    printf "SHEPHERD: ANALYSIS: %sTOTAL %0.2f%% %s %0.2f%%: %s\n", $statusstring, $total_data_percent,
+        ($total_data_percent >= $channel_ok_threshold_percent ? ">" : "<"), $channel_ok_threshold_percent,
+        ($retval ? "WANT MORE DATA" : "COMPLETE");
+    return $retval;
+}
+
+
+# work out date range we are expecting data to be in
+sub calc_date_range
+{
     # normalize starttime to beginning of hour
-    my ($sec,$min,@rest) = localtime($master_starttime);
-    my $starttime = $master_starttime - ((60 * $min) + $sec);
-    my $endtime;
+    my $now = time;
+    my ($sec,$min,@rest) = localtime($now);
+
+    $starttime = $now - ((60 * $min) + $sec);
 
     if ($days) {
@@ -243 +375 @@
         $endtime = $starttime + (7*86400);
     }
-    $starttime += (86400 * $opt->{offset};
-
-    # XXX TODO!  for now just return 0 - we got all our data!
-    return 0;
-
-    # iterate across channels
-    #foreach my $channel (sort keys %{$channels}) {
-        # iterate across days
-
-    return 0;
+    $starttime += (86400 * $opt->{offset}) if ($opt->{offset});
+
+    $num_timeslots = ($endtime - $starttime) / $timeslot_size;
 }
 

status

@@ -3 +3 @@
 ltd:1.51
 oztivo:0.2
-abc_website:1.52
+abc_website:1.53
 abc2_website:1.52
-yahoo7widget:1.52
+yahoo7widget:1.53