Changeset 156

Timestamp:

10/26/06 17:15:59 (7 years ago)

Author:

lincoln

Message:

1. log to stdout rather than stderr on sbsnews/abc_website/abc2_website/reconciler_mk2/imdb_augment data (means shepherd can log the output from these),
2. print explicit error/warning on invalid time from yahoo xml data
3. clean non-standard characters from abc_website/abc2_website data

Files:

• grabbers/abc2_website (modified) (7 diffs)
• grabbers/abc_website (modified) (7 diffs)
• grabbers/sbsnews_website.conf (modified) (1 diff)
• grabbers/yahoo7widget (modified) (5 diffs)
• postprocessors/imdb_augment_data (modified) (7 diffs)
• reconcilers/reconciler_mk2 (modified) (4 diffs)
• status (modified) (1 diff)

grabbers/abc2_website

@@ -18 +18 @@
 
 my $progname = "abc2_website";
-my $version = "1.57_20oct06";
+my $version = "1.57_27oct06";
 
 use LWP::UserAgent;
@@ -185 +185 @@
                 no warnings 'all'; eval <>; die "$@" if $@;
         } else {
-                printf STDERR "WARNING: no programme cache $opt_cache_file - have to fetch all details\n";
+                printf "WARNING: no programme cache $opt_cache_file - have to fetch all details\n";
 
                 # try to write to it - if directory doesn't exist this will then cause an error
@@ -198 +198 @@
 {
         if (!(open(F,">$opt_cache_file"))) {
-                printf STDERR "WARNING: could not write cache file $opt_cache_file: $!\n";
-                printf STDERR "Please fix this in order to reduce the number of queries for data!\n";
+                printf "WARNING: could not write cache file $opt_cache_file: $!\n";
+                printf "Please fix this in order to reduce the number of queries for data!\n";
                 sleep 10;
         } else {
@@ -357 +357 @@
                 }
                 if ($seen_programme == 0) {
-                        printf STDERR "WARNING: failed to parse any programme data from '%s' - blocked/rate-limited/format-changed?\n",$url;
+                        printf "WARNING: failed to parse any programme data from '%s' - blocked/rate-limited/format-changed?\n",$url;
                         $stats{failed_to_parse_portal_detail_page}++;
                 }
@@ -422 +422 @@
 {
         my ($entry) = @_;
-        printf STDERR "%s [%d] %s\n",$progname, time,$entry;
+        printf "%s [%d] %s\n",$progname, time,$entry;
 }
 
@@ -429 +429 @@
 sub print_stats
 {
-        printf STDERR "%s %s [%d] completed in %0.2f seconds",$progname, $version, time, tv_interval($script_start_time);
+        printf "%s %s [%d] completed in %0.2f seconds",$progname, $version, time, tv_interval($script_start_time);
         foreach my $key (sort keys %stats) {
-                printf STDERR ", %d %s",$stats{$key},$key;
-        }
-        printf STDERR "\n";
+                printf ", %d %s",$stats{$key},$key;
+        }
+        printf "\n";
 }
 
@@ -451 +451 @@
         elsif (defined $$x) {
                 $$x =~ s/&(#(\d+)|(.*?));/ $2 ? chr($2) : $amp{$3}||' ' /eg;
-                # $$x =~ s/[^\x20-\x7f]/ /g;
+                $$x =~ s/[^\x20-\x7f]/ /g;
                 $$x =~ s/(^\s+|\s+$)//g;
         }

grabbers/abc_website

@@ -18 +18 @@
 
 my $progname = "abc_website";
-my $version = "1.57_21oct06";
+my $version = "1.57_27oct06";
 
 use LWP::UserAgent;
@@ -185 +185 @@
                 no warnings 'all'; eval <>; die "$@" if $@;
         } else {
-                printf STDERR "WARNING: no programme cache $opt_cache_file - have to fetch all details\n";
+                printf "WARNING: no programme cache $opt_cache_file - have to fetch all details\n";
 
                 # try to write to it - if directory doesn't exist this will then cause an error
@@ -198 +198 @@
 {
         if (!(open(F,">$opt_cache_file"))) {
-                printf STDERR "WARNING: could not write cache file $opt_cache_file: $!\n";
-                printf STDERR "Please fix this in order to reduce the number of queries for data!\n";
+                printf "WARNING: could not write cache file $opt_cache_file: $!\n";
+                printf "Please fix this in order to reduce the number of queries for data!\n";
                 sleep 10;
         } else {
@@ -357 +357 @@
                 }
                 if ($seen_programme == 0) {
-                        printf STDERR "WARNING: failed to parse any programme data from '%s' - blocked/rate-limited/format-changed?\n",$url;
+                        printf "WARNING: failed to parse any programme data from '%s' - blocked/rate-limited/format-changed?\n",$url;
                         $stats{failed_to_parse_portal_detail_page}++;
                 }
@@ -422 +422 @@
 {
         my ($entry) = @_;
-        printf STDERR "%s [%d] %s\n",$progname, time,$entry;
+        printf "%s [%d] %s\n",$progname, time,$entry;
 }
 
@@ -429 +429 @@
 sub print_stats
 {
-        printf STDERR "%s %s [%d] completed in %0.2f seconds",$progname, $version, time, tv_interval($script_start_time);
+        printf "%s %s [%d] completed in %0.2f seconds",$progname, $version, time, tv_interval($script_start_time);
         foreach my $key (sort keys %stats) {
-                printf STDERR ", %d %s",$stats{$key},$key;
-        }
-        printf STDERR "\n";
+                printf ", %d %s",$stats{$key},$key;
+        }
+        printf "\n";
 }
 
@@ -451 +451 @@
         elsif (defined $$x) {
                 $$x =~ s/&(#(\d+)|(.*?));/ $2 ? chr($2) : $amp{$3}||' ' /eg;
-                # $$x =~ s/[^\x20-\x7f]/ /g;
+                $$x =~ s/[^\x20-\x7f]/ /g;
                 $$x =~ s/(^\s+|\s+$)//g;
         }

grabbers/sbsnews_website.conf

@@ -6 +6 @@
             'regions' => '',
             'option_ready' => '--version',
-            'desc' => 'guide data for SBS News only from http://www.sbs.com.au/whatson/WNC-Schedule.html',
+            'desc' => 'guide data for SBS News only from www.sbs.com.au/whatson/WNC-Schedule.html',
             'max_days' => 7,
             'category' => 2,

grabbers/yahoo7widget

@@ -17 +17 @@
 
 my $progname = "yahoo7widget";
-my $version = "1.56_21oct06";
+my $version = "1.57_21oct06";
 
 use LWP::UserAgent;
@@ -39 +39 @@
 
 my $ua;
-BEGIN {
-        $ua = LWP::UserAgent->new(
-                'timeout' => 30,
-                'keep_alive' => 1,
-                'agent' => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-us)'
-                );
-        $ua->env_proxy;
-        # $ua->cookie_jar({});
-        $| = 1;
-}
+$ua = LWP::UserAgent->new(
+        'timeout' => 30,
+        'keep_alive' => 1,
+        'agent' => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-us)'
+);
+$ua->env_proxy;
+# $ua->cookie_jar({});
+$| = 1;
 
 #
@@ -295 +293 @@
 {
         my ($entry) = @_;
-        printf STDERR "%s [%d] %s\n",$progname,time,$entry;
+        printf "%s [%d] %s\n",$progname,time,$entry;
 }
 
@@ -302 +300 @@
 sub print_stats
 {
-        printf STDERR "%s v%s [%d] completed in %0.2f seconds",$progname,$version,time,tv_interval($script_start_time);
+        printf "%s v%s [%d] completed in %0.2f seconds",$progname,$version,time,tv_interval($script_start_time);
         foreach my $key (sort keys %stats) {
-                printf STDERR ", %d %s",$stats{$key},$key;
-        }
-        printf STDERR "\n";
+                printf ", %d %s",$stats{$key},$key;
+        }
+        printf "\n";
 }
 
@@ -331 +329 @@
                         my $event_start =       $event->getElementsByTagName("event_date")->item(0)->getFirstChild->getNodeValue;
                         my $event_end =         $event->getElementsByTagName("end_date")->item(0)->getFirstChild->getNodeValue;
+
+                        if ($event_start < 10) {
+                                $stats{progs_with_invalid_start}++;
+                                printf "WARNING: programme with event_id '$event_id' had an invalid start time of '$event_start'; skipped\n";
+                                next;
+                        }
+                        if ($event_end < 10) {
+                                $stats{progs_with_invalid_end}++;
+                                printf "WARNING: programme with event_id '$event_id' had an invalid end time of '$event_end'; skipped\n";
+                                next;
+                        }
+
                         $event_id .= $event_start . $event_end; # event_id actually isn't unique - so make it so
 

postprocessors/imdb_augment_data

@@ -25 +25 @@
 
 my $progname = "imdb_augment_data";
-my $version = "0.03_19aug06";
+my $version = "0.04_19aug06";
 
 use LWP::UserAgent;
@@ -200 +200 @@
                 no warnings 'all'; eval <>; die "$@" if $@;
         } else {
-                printf STDERR "WARNING: no cache $opt->{cache_file} - ".
+                printf "WARNING: no cache $opt->{cache_file} - ".
                   "have to fetch all details.\n";
                 &write_cache; # try to write to it - failure will cause an error & barf
@@ -237 +237 @@
 {
         if (!(open(F,">$opt->{cache_file}"))) {
-                printf STDERR "ERROR: could not write cache file %s: %s\n",
+                printf "ERROR: could not write cache file %s: %s\n",
                   $opt->{cache_file}, $!;
-                printf STDERR "You need to fix this before you can use %s\n",
+                printf "You need to fix this before you can use %s\n",
                   $progname;
                 exit(1);
@@ -309 +309 @@
 {
         my ($entry) = @_;
-        printf STDERR "%s [%d] %s\n",$progname, time,$entry;
+        printf "%s [%d] %s\n",$progname, time,$entry;
 }
 
@@ -316 +316 @@
 sub print_stats
 {
-        printf STDERR "%s v%s [%d] completed in %0.2f seconds",
+        printf "%s v%s [%d] completed in %0.2f seconds",
           $progname, $version, time, tv_interval($script_start_time);
         foreach my $key (sort keys %stats) {
-                printf STDERR ", %d %s",$stats{$key},$key;
-        }
-        printf STDERR "\n";
+                printf ", %d %s",$stats{$key},$key;
+        }
+        printf "\n";
 
         if ($opt->{debug}) {
@@ -714 +714 @@
         }
 
-        &log("looking up \"$movie_title\" duration $prog_duration, matching categories: $interested");
         $stats{included_for_imdb_lookup}++;
 
@@ -735 +734 @@
         if (defined $data_cache->{movie_id_lookup}->{$post_fields}) {
                 $stats{imdb_lookup_used_cache_entry}++;
+                &log("used (cache) lookup \"$movie_title\" duration $prog_duration, matching categories: $interested");
         } else {
+                &log("looking up \"$movie_title\" duration $prog_duration, matching categories: $interested");
                 &search_imdb_online($movie_title,$post_fields);
 

reconcilers/reconciler_mk2

@@ -91 +91 @@
 
 my $progname = "reconciler_mk2";
-my $version = "0.09_24oct06";
+my $version = "0.10_27oct06";
 
 use LWP::UserAgent;
@@ -420 +420 @@
         # check if we are in an eval()
         if ($^S) {
-                printf STDERR "  caught a die() within eval{} from file $file line $line\n";
+                printf "  caught a die() within eval{} from file $file line $line\n";
         } else {
                 if (!ref($arg)) {
-                        printf STDERR "DIE at line %d in file %s:\nERROR: %s\n",$line,$file,$arg;
+                        printf "DIE at line %d in file %s:\nERROR: %s\n",$line,$file,$arg;
                         CORE::die(join("",@rest));
                 } else {
@@ -505 +505 @@
 {
         my ($log_level,$entry) = @_;
-        printf STDERR "%s [%d] %s\n",$progname, time,$entry if ($log_level);
+        printf "%s [%d] %s\n",$progname, time,$entry if ($log_level);
         #printf LOG_FILE "%s [%d] %s\n",$progname, time,$entry if (($log_level) && (!$opt->{nolog}));
         printf LOG_FILE "%s [%d] %s\n",$progname, time,$entry unless ($opt->{nolog});
@@ -514 +514 @@
 sub print_stats
 {
-        printf STDERR "%s v%s [%d] completed in %0.2f seconds",$progname, $version, time, tv_interval($script_start_time);
+        printf "%s v%s [%d] completed in %0.2f seconds",$progname, $version, time, tv_interval($script_start_time);
         foreach my $key (sort keys %stats) {
-                printf STDERR ",\n\t%d %s", $stats{$key},$key;
-        }
-        printf STDERR ".\n";
+                printf ",\n\t%d %s", $stats{$key},$key;
+        }
+        printf ".\n";
 }
 

status

@@ -4 +4 @@
 grabber:abc_website:1.57-r1
 grabber:abc2_website:1.57-r1
-grabber:sbsnews_website:0.02
-grabber:yahoo7widget:1.56-r2
+grabber:sbsnews_website:0.02-r1
+grabber:yahoo7widget:1.57
 grabber:jrobbo:0.04-r3
 grabber:d1:0.6.2.4-r2
 reconciler:reconciler_mk2:0.09-r1
-postprocessor:imdb_augment_data:0.03-r1
+postprocessor:imdb_augment_data:0.04