Changeset 783

Timestamp:

06/20/07 05:45:15 (6 years ago)

Author:

paul

Message:

abc_website: limit detail page failures

Files:

• grabbers/abc_website (modified) (3 diffs)
• status (modified) (1 diff)

grabbers/abc_website

@@ -490 +490 @@
                 } else {
                         if ((!$opt_cheap) && ($unprocessed_url[$i] ne "")) {
-                                $stats{portal_detail_pages}++;
                                 &get_one_abc_event($cache_key, $unprocessed_url[$i]);
 
@@ -531 +530 @@
 {
         my ($cache_key, $url) = @_;
+
+        if ($stats{failed_to_fetch_portal_detail_page} >= 3 or $stats{failed_to_parse_portal_detail_page} >= 9)
+        {
+                &log("Skipping detail page because too many failures.");
+                return;
+        }
+
         my $seen_programme = 0;
-
         my $data = Shepherd::Common::get_url(url => $url, debug => $debug);
         my $tree = HTML::TreeBuilder->new_from_content($data) if ($data);
         if (!defined $tree) {
                 &log("failed to fetch $url; skipping");
+                $stats{failed_to_fetch_portal_detail_page}++;
                 return;
         }
@@ -621 +627 @@
                 printf "WARNING: failed to parse any programme data from '%s' - blocked/rate-limited/format-changed?\n",$url;
                 $stats{failed_to_parse_portal_detail_page}++;
-        }
+                return;
+        }
+        $stats{portal_detail_pages}++;
 }
 

status

@@ -5 +5 @@
 grabber         yahoo7widget        1.86
 grabber         rex                 3.5.5
-grabber         abc_website         3.04
+grabber         abc_website         3.05
 grabber         sbsnews_website     0.16
 grabber         oztivo              1.00