Changeset 826

Timestamp:

07/01/07 05:15:10 (6 years ago)

Author:

paul

Message:

abc_website: remove some unused 'use' statements, set defaults for Shepherd::Common, minor message changes, use Shepherd::Common::print_stats, wait an extra 2 days before clearing cache, simplify what days to grab because wasn't helpful, removed the insertion of fake 'Station Close' and replaced with a cached version of the true 'Station Close' (wont work the first day because nothing yet cached), only return programmes for requested days, use Shepherd::Common::log for all logs except printf, still has gaps for abc2 on days beyond weekly 'Station Close' data

Files:

• grabbers/abc_website (modified) (18 diffs)
• status (modified) (1 diff)

grabbers/abc_website

@@ -23 +23 @@
 
 my $progname = "abc_website";
-my $version = "3.08";
-
-use LWP::UserAgent;
+my $version = "3.09";
+
 use XMLTV;
 use POSIX qw(strftime mktime);
 use Getopt::Long;
 use HTML::TreeBuilder;
-use Data::Dumper;
 use Storable;
 use Shepherd::Common;
@@ -53 +51 @@
 my $data_cache;
 my $override_settings = { };
-my $station_close_data;
 my $writer;
 $| = 1;
@@ -121 +118 @@
 die "no channel file specified, see --help for instructions\n", if ($opt_channels_file eq "");
 
+# set defaults
+&Shepherd::Common::set_default("debug", (defined $debug ? ($debug * 2) : 0));
+&Shepherd::Common::set_defaults(stats => \%stats, referer => "last");
+
 #
 # go go go!
 #
 
-my $starttime = time;
 &read_cache if ($opt_no_cache == 0);
 
@@ -135 +135 @@
 die "nothing to do; neither ABC nor ABC2 in channels lineup!\n" if ((!defined $channels->{ABC}) && (!defined $channels->{ABC2}));
 
-&log(sprintf "going to %s%s %s%d%s days%s of data for ABC(%s), ABC2(%s) into %s (%s)",
+&log(sprintf "Going to %s%s %s%d%s days%s of data for ABC(%s), ABC2(%s) into %s (%s)",
         ($opt_gaps_file ne "" ? "micro-gap " : ""),
         ($opt_cheap ? "verify (cache-validate)" : "grab"),
@@ -177 +177 @@
 $writer->end;
 
-&print_stats;
+&Shepherd::Common::print_stats($progname, $version, $script_start_time, %stats);
 exit(0);
 
@@ -267 +267 @@
         for my $cache_key (keys %{$data_cache}) {
                 my ($starttime, @rest) = split(/,/,$cache_key);
-                if ($starttime < (time-86400)) {
+                if ($starttime < ($script_start_time - (2*24*60*60))) {
                         delete $data_cache->{$cache_key};
                         $stats{removed_items_from_cache}++;
@@ -291 +291 @@
         my @gap_s, my @gap_e;
 
-        my $daynum = $opt_offset;
-        $opt_days = 28 if (($opt_do_extra_days) && ($opt_gaps_file eq "") && ($opt_offset == 0) && ($opt_days == 7));
-        my $days_left = $opt_days - $opt_offset;
-
-        my @timeattr = localtime($starttime); # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
-        # if now before 4:30am we need the previous day due to the guide starting at 4:30am
-        if (($daynum == 0 and ($timeattr[2] < 4 or ($timeattr[2] == 4 and $timeattr[1] < 30))) or $daynum < 0) {
-                $daynum = -1;
-                $days_left = $opt_days + 1;
-        } elsif ($daynum > 0) {
-                $daynum--;
-                $days_left++;
-        }
+        $opt_days = 28 if ($opt_do_extra_days);
+
+        # guide pages start at 4:30am so grab day before
+        my $daynum = $opt_offset - 1;
+        my $days_left = $opt_days - $opt_offset + 1;
+
+        my @timeattr = localtime($script_start_time); # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
         $timeattr[0] = 0; # zero seconds
         $timeattr[3] += $daynum - 1; # day
@@ -311 +305 @@
                 $daynum++;
 
+                # guide page starts at 4:30am
                 $timeattr[1] = 30; # minutes
                 $timeattr[2] = 4; # hours 4:30am
@@ -339 +334 @@
                 &log((sprintf "Fetching %s summary data: day %d of %d",
                         $xmlid, $daynum, $opt_days ));
-                my $data = Shepherd::Common::get_url(url => $url, retries => ($tries-1), debug => $debug * 2);
+                my $data = Shepherd::Common::get_url(url => $url, retries => ($tries-1));
                 my $tree;
                 $tree = HTML::TreeBuilder->new_from_content($data) if ($data);
@@ -394 +389 @@
                                                                                 $daynum,$tree_row->as_text(),POSIX::strftime("%Y%m%d%H%M", localtime($found_time)),
                                                                                 $programme,$progurl if ($debug && $debug > 1);
-                                                                        # If it's day 1 and the first programme found 
-                                                                        # starts later than now, insert a Station Close 
-                                                                        # from now until then. Without this, we have a
-                                                                        # hole if run during a Station Close period 
-                                                                        # (eg midnight - 6am).
-                                                                        if (!$seen_programmes and $daynum == 1 and $found_time > $script_start_time)
-                                                                        {
-                                                                            &log("Inserting initial Station Close");
-                                                                            $unprocessed_progname[$unprocessed_programmes] = "Station Close";
-                                                                            $unprocessed_starttime[$unprocessed_programmes] = $script_start_time;
-                                                                            $unprocessed_day[$unprocessed_programmes] = $daynum;
-                                                                            $unprocessed_url[$unprocessed_programmes] = "";
-                                                                            $unprocessed_programmes++;
-                                                                        }
 
                                                                         $unprocessed_progname[$unprocessed_programmes] = $programme;
@@ -431 +412 @@
                         $stats{abc_daily_pages}++;
 
-                        if ((defined $station_close_data) && (defined $station_close_data->{$xmlid}) &&
-                            (defined $station_close_data->{$xmlid}->[$daynum])) {
+                        my $cache_key = sprintf "%d,%d,%s,%s", $currtime, 0, $xmlid, "Station Close";
+                        if (defined $data_cache->{$cache_key}) {
                                 # get station-close time from the previously-fetched "weekly programme guide"
-                                &log("inserting Station Close");
+                                &log(" Inserting 'Station Close'");
 
                                 $unprocessed_progname[$unprocessed_programmes] = "Station Close";
-                                $unprocessed_starttime[$unprocessed_programmes] = $station_close_data->{$xmlid}->[$daynum];
+                                $unprocessed_starttime[$unprocessed_programmes] = $data_cache->{$cache_key};
                                 $unprocessed_day[$unprocessed_programmes] = $daynum;
                                 $unprocessed_url[$unprocessed_programmes] = "";
@@ -453 +434 @@
 
         # have 'n' days of this channel unprocessed - process it!
-        &log((sprintf "have summary data for %s, now fetching detail pages for up to %d programmes..",$chan_id,$unprocessed_programmes-2));
+        &log((sprintf "Have summary data for %s, now fetching detail pages for up to %d programmes..",$chan_id,$unprocessed_programmes-2));
+
+        # only return programmes for requested days
+        @timeattr = localtime($script_start_time); # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
+        $timeattr[0] = 0; # zero seconds
+        $timeattr[1] = 0; # zero mintues
+        $timeattr[2] = 0; # zero hours
+        $timeattr[3] += $opt_offset; # day
+        my $start_day = mktime(@timeattr);
+        $timeattr[3] += $opt_days - $opt_offset; # day
+        my $stop_day = mktime(@timeattr);
 
         for (my $i = 0; $i < ($unprocessed_programmes-1); $i++) {
@@ -496 +487 @@
                         printf "gap-fetching: including prog '%s', start %d, end %d\n", $unprocessed_progname[$i], 
                                 $unprocessed_starttime[$i], $unprocessed_starttime[$i+1] if $debug;
+                } elsif ($unprocessed_starttime[$i+1] <= $start_day or
+                                        $unprocessed_starttime[$i] >= $stop_day) {
+                        printf "skipping program because before or after requested days, '%s', start %d, end %d\n",
+                                         $unprocessed_progname[$i], $unprocessed_starttime[$i], $unprocessed_starttime[$i+1] if $debug;
+                        next;
                 }
 
@@ -555 +551 @@
         if ($stats{failed_to_fetch_portal_detail_page} >= 3 or $stats{failed_to_parse_portal_detail_page} >= 9)
         {
-                &log("Skipping detail page because too many failures.");
+                &log("skipping detail page because too many failures.");
                 return;
         }
 
         my $seen_programme = 0;
-        my $data = Shepherd::Common::get_url(url => $url, debug => $debug);
+        my $data = Shepherd::Common::get_url(url => $url);
         my $tree = HTML::TreeBuilder->new_from_content($data) if ($data);
         if (!defined $tree) {
@@ -658 +654 @@
 sub log
 {
-        my ($entry) = @_;
-        printf "%s\n", $entry;
-}
-
-######################################################################################################
-
-sub print_stats
-{
-        printf "STATS: %s v%s completed in %d seconds", $progname, $version, (time-$script_start_time);
-        foreach my $key (sort keys %stats) {
-                printf ", %d %s",$stats{$key},$key;
-        }
-        printf "\n";
+        &Shepherd::Common::log(@_);
 }
 
@@ -678 +662 @@
 {
         my ($xmlid,$url) = @_;
-        &log("fetching (weekly) station close data for $xmlid");
-        my $data = Shepherd::Common::get_url(url => $url, debug => $debug);
+        &log("Fetching weekly 'Station Close' data for $xmlid");
+        my $data = Shepherd::Common::get_url(url => $url);
         my $tree = HTML::TreeBuilder->new_from_content($data) if ($data);
 
@@ -695 +679 @@
                         if (defined $last_td_text) {
                                         # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
-                                        my @timeattr = localtime($starttime + (($daynum-1)*86400));
+                                        my @timeattr = localtime($script_start_time + (($daynum-1)*86400));
                                         $timeattr[0] = 0; # zero seconds
 
@@ -717 +701 @@
                                                 }
 
-                                                $station_close_data->{$xmlid}->[$daynum] = $found_time;
+                                                $timeattr[1] = 30; # min
+                                                $timeattr[2] = 4; # hours 4:30am
+                                                my $currtime = mktime(@timeattr);
+                                                my $cache_key = sprintf "%d,%d,%s,%s", $currtime, 0, $xmlid, "Station Close";
+                                                $data_cache->{$cache_key} = $found_time;
 
                                                 printf "station close time for %s day %d is %s\n",

status

@@ -5 +5 @@
 grabber         yahoo7widget        1.88-r2
 grabber         rex                 3.5.5-r2
-grabber         abc_website         3.08
+grabber         abc_website         3.09
 grabber         sbsnews_website     0.18
 grabber         oztivo              1.07