Changeset 667

Timestamp:

05/22/07 19:39:13 (6 years ago)

Author:

lincoln

Message:

fix regression in abc/abc2_website: when fetching extra days, only try to fetch daily page once for beyond 7 days. also put parse_xmltv_date and cleanup into common library code

Files:

• grabbers/abc2_website (modified) (5 diffs)
• grabbers/abc_website (modified) (5 diffs)
• references/Shepherd/Common.pm (modified) (2 diffs)
• status (modified) (2 diffs)

grabbers/abc2_website

@@ -22 +22 @@
 my $progname = "abc2_website";
 my $chan_id = "ABC2";
-my $version = "2.12";
+my $version = "2.13";
 
 use LWP::UserAgent;
@@ -47 +47 @@
 
 my $script_start_time = time;
-my $gmt_offset;
 my %stats;
 my $channels, my $opt_channels, my $gaps;
@@ -342 +341 @@
                 &log((sprintf "Fetching %s summary data: day %d of %d",
                         $xmlid, $daynum, $opt_days ));
-                my $data = Shepherd::Common::get_url(url => $url, retries => 4, debug => $debug * 2);
+                my $data = Shepherd::Common::get_url(url => $url, retries => $tries, debug => $debug * 2);
                 my $tree = HTML::TreeBuilder->new_from_content($data) if ($data);
 
@@ -506 +505 @@
                 }
 
-                &cleanup($prog);
+                Shepherd::Common::cleanup($prog);
                 $writer->write_programme($prog);
         }
@@ -599 +598 @@
         }
         printf "\n";
-}
-
-######################################################################################################
-# descend a structure and clean up various things, including stripping
-# leading/trailing spaces in strings, translations of html stuff etc
-#   -- taken & modified from Michael 'Immir' Smith's excellent tv_grab_au
-
-sub cleanup {
-        my $x = shift;
-        if    (ref $x eq "REF")   { cleanup($_) }
-        elsif (ref $x eq "HASH")  { cleanup(\$_) for values %$x }
-        elsif (ref $x eq "ARRAY") { cleanup(\$_) for @$x }
-        elsif (defined $$x) {
-                $$x =~ s/&(#(\d+)|(.*?));/ $2 ? chr($2) : $amp{$3}||' ' /eg;
-                $$x =~ s/[^\x20-\x7f]/ /g;
-                $$x =~ s/(^\s+|\s+$)//g;
-        }
-}
-
-######################################################################################################
-
-# strptime type date parsing - BUT - if no timezone is present, treat time as being in localtime
-# rather than the various other perl implementation which treat it as being in UTC/GMT
-sub parse_xmltv_date
-{
-        my $datestring = shift;
-        my @t; # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
-        my $tz_offset = 0;
-
-        # work out GMT offset - we only do this once
-        if (!$gmt_offset) {
-                my $tzstring = strftime("%z", localtime(time));
-
-                $gmt_offset = (60*60) * int(substr($tzstring,1,2));     # hr
-                $gmt_offset += (60 * int(substr($tzstring,3,2)));       # min
-                $gmt_offset *= -1 if (substr($tzstring,0,1) eq "-");    # +/-
-        }
-
-        if ($datestring =~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})/) {
-                ($t[5],$t[4],$t[3],$t[2],$t[1],$t[0]) = (int($1)-1900,int($2)-1,int($3),int($4),int($5),0);
-                ($t[6],$t[7],$t[8]) = (-1,-1,-1);
-
-                # if input data has a timezone offset, then offset by that
-                if ($datestring =~ /\+(\d{2})(\d{2})/) {
-                        $tz_offset = $gmt_offset - (($1*(60*60)) + ($2*60));
-                } elsif ($datestring =~ /\-(\d{2})(\d{2})/) {
-                        $tz_offset = $gmt_offset + (($1*(60*60)) + ($2*60));
-                }
-
-                my $e = mktime(@t);
-                return ($e+$tz_offset) if ($e > 1);
-        }
-        return undef;
 }
 

grabbers/abc_website

@@ -22 +22 @@
 my $progname = "abc_website";
 my $chan_id = "ABC";
-my $version = "2.12";
+my $version = "2.13";
 
 use LWP::UserAgent;
@@ -47 +47 @@
 
 my $script_start_time = time;
-my $gmt_offset;
 my %stats;
 my $channels, my $opt_channels, my $gaps;
@@ -342 +341 @@
                 &log((sprintf "Fetching %s summary data: day %d of %d",
                         $xmlid, $daynum, $opt_days ));
-                my $data = Shepherd::Common::get_url(url => $url, retries => 4, debug => $debug * 2);
+                my $data = Shepherd::Common::get_url(url => $url, retries => $tries, debug => $debug * 2);
                 my $tree = HTML::TreeBuilder->new_from_content($data) if ($data);
 
@@ -506 +505 @@
                 }
 
-                &cleanup($prog);
+                Shepherd::Common::cleanup($prog);
                 $writer->write_programme($prog);
         }
@@ -599 +598 @@
         }
         printf "\n";
-}
-
-######################################################################################################
-# descend a structure and clean up various things, including stripping
-# leading/trailing spaces in strings, translations of html stuff etc
-#   -- taken & modified from Michael 'Immir' Smith's excellent tv_grab_au
-
-sub cleanup {
-        my $x = shift;
-        if    (ref $x eq "REF")   { cleanup($_) }
-        elsif (ref $x eq "HASH")  { cleanup(\$_) for values %$x }
-        elsif (ref $x eq "ARRAY") { cleanup(\$_) for @$x }
-        elsif (defined $$x) {
-                $$x =~ s/&(#(\d+)|(.*?));/ $2 ? chr($2) : $amp{$3}||' ' /eg;
-                $$x =~ s/[^\x20-\x7f]/ /g;
-                $$x =~ s/(^\s+|\s+$)//g;
-        }
-}
-
-######################################################################################################
-
-# strptime type date parsing - BUT - if no timezone is present, treat time as being in localtime
-# rather than the various other perl implementation which treat it as being in UTC/GMT
-sub parse_xmltv_date
-{
-        my $datestring = shift;
-        my @t; # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
-        my $tz_offset = 0;
-
-        # work out GMT offset - we only do this once
-        if (!$gmt_offset) {
-                my $tzstring = strftime("%z", localtime(time));
-
-                $gmt_offset = (60*60) * int(substr($tzstring,1,2));     # hr
-                $gmt_offset += (60 * int(substr($tzstring,3,2)));       # min
-                $gmt_offset *= -1 if (substr($tzstring,0,1) eq "-");    # +/-
-        }
-
-        if ($datestring =~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})/) {
-                ($t[5],$t[4],$t[3],$t[2],$t[1],$t[0]) = (int($1)-1900,int($2)-1,int($3),int($4),int($5),0);
-                ($t[6],$t[7],$t[8]) = (-1,-1,-1);
-
-                # if input data has a timezone offset, then offset by that
-                if ($datestring =~ /\+(\d{2})(\d{2})/) {
-                        $tz_offset = $gmt_offset - (($1*(60*60)) + ($2*60));
-                } elsif ($datestring =~ /\-(\d{2})(\d{2})/) {
-                        $tz_offset = $gmt_offset + (($1*(60*60)) + ($2*60));
-                }
-
-                my $e = mktime(@t);
-                return ($e+$tz_offset) if ($e > 1);
-        }
-        return undef;
 }
 

references/Shepherd/Common.pm

@@ -16 +16 @@
 use Compress::Zlib;
 use Data::Dumper;
-
-my $version = '0.6';
-
+use POSIX qw(strftime mktime);
+
+my $version = '0.7';
+my $gmt_offset;
 my $ua;
 
@@ -237 +238 @@
 }
 
+##########################################################################
+# descend a structure and clean up various things, including stripping
+# leading/trailing spaces in strings, translations of html stuff etc
+#   -- taken & modified from Michael 'Immir' Smith's excellent tv_grab_au
+
+sub cleanup {
+    my $x = shift;
+    if    (ref $x eq "REF")   { cleanup($_) }
+    elsif (ref $x eq "HASH")  { cleanup(\$_) for values %$x }
+    elsif (ref $x eq "ARRAY") { cleanup(\$_) for @$x }
+    elsif (defined $$x) {
+        $$x =~ s/&(#(\d+)|(.*?));/ $2 ? chr($2) : $amp{$3}||' ' /eg;
+        $$x =~ s/[^\x20-\x7f]/ /g;
+        $$x =~ s/(^\s+|\s+$)//g;
+    }
+}
+
+
+##########################################################################
+# strptime type date parsing - BUT - if no timezone is present, treat
+# time as being in localtime rather than the various other perl
+# implementation which treat it as being in UTC/GMT
+
+sub parse_xmltv_date
+{
+    my $datestring = shift;
+    my @t; # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
+    my $tz_offset = 0;
+
+    # work out GMT offset - we only do this once
+    if (!$gmt_offset) {
+        my $tzstring = strftime("%z", localtime(time));
+
+        $gmt_offset = (60*60) * int(substr($tzstring,1,2));     # hr
+        $gmt_offset += (60 * int(substr($tzstring,3,2)));       # min
+        $gmt_offset *= -1 if (substr($tzstring,0,1) eq "-");    # +/-
+    }
+
+    if ($datestring =~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})/) {
+        ($t[5],$t[4],$t[3],$t[2],$t[1],$t[0]) = (int($1)-1900,int($2)-1,int($3),int($4),int($5),0);
+        ($t[6],$t[7],$t[8]) = (-1,-1,-1);
+
+        # if input data has a timezone offset, then offset by that
+        if ($datestring =~ /\+(\d{2})(\d{2})/) {
+            $tz_offset = $gmt_offset - (($1*(60*60)) + ($2*60));
+        } elsif ($datestring =~ /\-(\d{2})(\d{2})/) {
+            $tz_offset = $gmt_offset + (($1*(60*60)) + ($2*60));
+        }
+
+        my $e = mktime(@t);
+        return ($e+$tz_offset) if ($e > 1);
+    }
+    return undef;
+}
+
+
 1;

status

@@ -2 +2 @@
 grabber         yahoo7widget        1.83-r1
 grabber         rex                 3.4.10
-grabber         abc_website         2.12
-grabber         abc2_website        2.12
+grabber         abc_website         2.13
+grabber         abc2_website        2.13
 grabber         sbsnews_website     0.14
 grabber         oztivo              0.28
@@ -16 +16 @@
 postprocessor   augment_timezone    0.12
 reference       channel_list        2
-reference       Shepherd/Common.pm  0.6
+reference       Shepherd/Common.pm  0.7