Changeset 602

Timestamp:

03/06/07 18:18:26 (6 years ago)

Author:

max

Message:

Rex handles network flakiness better

Files:

• grabbers/rex (modified) (7 diffs)
• status (modified) (1 diff)

grabbers/rex

@@ -3 +3 @@
 # "Rex"
 
-my $version  = '3.5.0';
+my $version  = '3.4.6';
 
 # An Australian TV Guide Grabber (a.k.a. tv_grab_au)
@@ -77 +77 @@
 # 3.4.4   : Bugfix : --dump-cache works
 # 3.4.5   : Bugfix : support regions with multiple channels of same name
+# 3.4.6   : Bugfix : more reliable dying on network failure
 
 use strict;
@@ -443 +444 @@
 
   # Set initial cookie
-  get_page($DATASOURCE);
+  unless (get_page($DATASOURCE, 5)) # Try hard to fetch main page
+  {
+      print "Unable to fetch main datasource page. No connectivity? Exiting.\n";
+      print stats(1);
+      die "Can't fetch main page.";
+  }
 
   # Set region/service cookie
-  post_page($DATASOURCE_SETUP,
-        [ 'fta_region_id' => $opt->{'region'} ]);
+  unless (post_page($DATASOURCE_SETUP, [ 'fta_region_id' => $opt->{'region'} ]))
+  {
+      print "ERROR: Unable to set region/service cookie.\n";
+      print stats(1);
+      die "Can't set region/service cookie.";
+  }
 
   $ua->cookie_jar()->scan(\&refresh_sid);
@@ -577 +587 @@
 sub get_page
 {
-  my ($url) = @_;
+  my ($url, $retries) = @_;
   my $request = GET $url;
-  return fetch_page($request);
+  return fetch_page($request, $retries);
 }
 
@@ -591 +601 @@
 sub fetch_page
 {
-  my ($request) = @_;
+  my ($request, $retries) = @_;
   
   $request->uri() =~ s/^http:\/\//$WW/ if $opt->{warper};
@@ -598 +608 @@
 
   print "Fetching: " . $request->as_string() . "\n" if ($debug);
+  $retries ||= 2;
   my $response;
-  for my $c (1..2) {
+  for my $c (1..$retries) {
     print "Attempt #$c.\n" if ($debug);
     $response = $ua->request($request);
@@ -615 +626 @@
         while ($r) { print "    ", $r->base, "\n"; $r = $r->previous }
     }
-    # Network down
-    if ($count_bad > 10 and $count_dl == 0)
-    {
-        print "ERROR! Unable to download anything useful. Smells like a " .
-              "network problem. Exiting.\n";
+    if ($count_dl and ($count_bad > ($count_dl * 3)))
+    {
+        print "Too many network failures. Exiting.\n";
         print stats(1);
-        die "No connectivity";
+        die "Too many network failures.";
     }
     return undef;

status

@@ -1 +1 @@
 application     shepherd            0.4.76
 grabber         yahoo7widget        1.81
-grabber         rex                 3.4.5
+grabber         rex                 3.4.6
 grabber         abc_website         2.11
 grabber         abc2_website        2.11