Changeset 7

Timestamp:

10/04/06 00:32:07 (7 years ago)

Author:

lincoln

Message:

improve retry logic for abc2_website grabbing - sometimes their website fails, use a variable # of retries depending on whether we are speculatively trying to get data further out than 7 days

Files:

• grabbers/abc2_website (modified) (7 diffs)

grabbers/abc2_website

@@ -16 +16 @@
 
 my $progname = "abc2_website";
-my $version = "1.53_03oct06";
+my $version = "1.54_03oct06";
 
 use LWP::UserAgent;
@@ -237 +237 @@
         my $stop_fetching = 0;
         my @unprocessed_progname, my @unprocessed_starttime, my @unprocessed_url;
+        my $daynum = 0;
 
         for (my $currtime = $starttime; $currtime < $endtime; $currtime += 86400) {
+                $daynum++;
                 # for abc portal data, treat a faulure as a hint that there is no further data.
                 # sometimes they have as much as 30 days of data ahead.  sometimes much less...
@@ -246 +248 @@
                         my $url = sprintf "%s/%s.htm",$urlbase,(strftime "%Y%m/%Y%m%d",localtime($currtime));
                         my $status = sprintf "%s summary data: day %d of %d", $xmlid, ((($currtime-$starttime)/86400)+1),(($endtime-$starttime)/86400);
-                        my $data = &get_url($url,$status,1);
+                        my $data = &get_url($url,$status,($daynum < 8 ? 5 : 2));
                         my $seen_programmes = 0;
 
@@ -318 +320 @@
 
         do {
-                my $data = &get_url($url,$status);
+                my $data = &get_url($url,$status,3);
 
                 my $tree = HTML::TreeBuilder->new_from_content($data);
@@ -368 +370 @@
 sub get_url
 {
-        my ($url,$status,$dontretry) = @_;
+        my ($url,$status,$retrycount) = @_;
         my $response;
         my $attempts = 0;
         my ($raw, $page, $base);
 
+        $retrycount = 5 if ($retrycount == 0);
         $url =~ s#^http://#http://webwarper.net/ww/# if $opt_warper;
         my $request = HTTP::Request->new(GET => $url);
@@ -383 +386 @@
         }
         &log(sprintf "fetching %s%s: %s",$status,($opt_obfuscate ? "[obfuscate]" : ""),$url);
-        for (1..3) {
+        for (1..$retrycount) {
                 $response = $ua->request($request);
-                last if ($response->is_success || $dontretry);
+                last if $response->is_success;
 
                 $stats{http_failed_requests}++;
@@ -393 +396 @@
         }
         if (!($response->is_success)) {
-                if ($dontretry == 0) {
-                        &log("aborting after $attempts attempts to fetch url $url") if $debug;
-                        printf STDERR "ERROR: could not open url %s in %d attempts\n",$url,$attempts;
-                }
+                &log("aborting after $attempts attempts to fetch url $url") if ($debug && $retrycount > 3);
                 return undef;
         }