Changeset 340

Timestamp:

12/04/06 18:53:28 (7 years ago)

Author:

max

Message:

Better handling of failed downloads and parses.

Files:

• grabbers/rex (modified) (17 diffs)
• status (modified) (1 diff)

grabbers/rex

@@ -3 +3 @@
 # "Rex"
 
-my $version  = '3.3.7';
+my $version  = '3.3.8';
 
 # An Australian TV Guide Grabber (a.k.a. tv_grab_au)
@@ -69 +69 @@
 # 3.3.6   : Bugfix : neater options parsing
 # 3.3.7   : Bugfix : now runs from current dir, not ~/.rex/
+# 3.3.8   : Bugfix : better handling of failed downloads & parses
 
 use strict;
@@ -92 +93 @@
 my $lang = "en";
 
-my ($count_dl, $count_detail, $count_bad, $count_cache, $count_changes, $count_kb) = (0) x 6;
+my ($count_dl, $count_detail, $count_bad, $count_cache, $count_changes, $count_kb, $count_bad_parse) = (0) x 7;
 
 my $DATASOURCE             = "http://www.yourtv.com.au";
@@ -213 +214 @@
 
       $guidedata = get_page($DATASOURCE_GUIDE_TODAY);
-      parse_guide($guidedata, $date);
+      parse_guide($guidedata, $date) if ($guidedata);
     }
     else
@@ -228 +229 @@
                           'submit' => 'submit'
                         ]);
-        parse_guide($guidedata, $date, $_);
+        parse_guide($guidedata, $date, $_) if ($guidedata);
       }
     }
@@ -346 +347 @@
   unless ($detailsdata and $result)
   {
-    print "Download failed.\n" if ($debug);
     sleep(5);
-    $count_bad++;
     refresh_ua();
     return download_show($pid, $recurse_count+1);
@@ -428 +427 @@
 
   # Set initial cookie
-  get_page($DATASOURCE, 1);
+  get_page($DATASOURCE);
 
   # Set region/service cookie
   post_page($DATASOURCE_SETUP,
-        [ 'fta_region_id' => $opt->{'region'} ],
-        1 );
+        [ 'fta_region_id' => $opt->{'region'} ]);
+
   $ua->cookie_jar()->scan(\&refresh_sid);
 
@@ -521 +520 @@
         " %d shows grabbed\n" .
         " %d downloads, including %d detail pages (%d KB)\n" .
-        " %d cache hits, %d changes from cache, %d failed downloads\n",
+        " %d cache hits, %d changes from cache\n" .
+        " %d failed downloads, %d failed parses\n",
         scalar(keys %shows),
         $count_dl, $count_detail, $count_kb, 
-        $count_cache, $count_changes, $count_bad);
+        $count_cache, $count_changes, 
+        $count_bad, $count_bad_parse);
   $ret .= " Time elapsed: " . timestats($t) . "\n";
   unless ($finished or !$count_detail)
@@ -553 +554 @@
 sub get_page
 {
-  my ($url, $ignore_failure) = @_;
+  my ($url) = @_;
   my $request = GET $url;
-  return fetch_page($request, $ignore_failure);
+  return fetch_page($request);
 }
 
 sub post_page
 {
-  my ($url, $headers, $ignore_failure) = @_;
+  my ($url, $headers) = @_;
   my $request = POST $url, $headers;
-  return fetch_page($request, $ignore_failure);
+  return fetch_page($request);
 }
 
 sub fetch_page
 {
-  my ($request, $ignore_failure) = @_;
+  my ($request) = @_;
   
   $request->uri() =~ s/^http:\/\//$WW/ if $opt->{warper};
@@ -578 +579 @@
     print "Attempt #$c.\n" if ($debug);
     $response = $ua->request($request);
-    last if ($response->is_success() or $ignore_failure);
+    last unless ($response->is_error());
+    $count_bad++;
+    print stats() if ($debug);
     sleep 5;
   }
-  unless ($response->is_success() or $ignore_failure)
+  if ($response->is_error())
   {
     print "ERROR! Failed to retrieve page: " . $request->uri() . ".\n";
-  }
-  if ($debug and (my $r = $response)->previous) 
-  {
-    print "GET_CONTENT_BASE redirection backtrace:\n";
-    while ($r) { print "    ", $r->base, "\n"; $r = $r->previous }
+    if ($debug and (my $r = $response)->previous) 
+    {
+        print "GET_CONTENT_BASE redirection backtrace:\n";
+        while ($r) { print "    ", $r->base, "\n"; $r = $r->previous }
+    }
+    # Network down
+    if ($count_bad > 10 and $count_dl == 0)
+    {
+        print "ERROR! Unable to download anything useful. Smells like a " .
+              "network problem. Exiting.\n";
+        print stats(1);
+        exit 1;
+    }
+    return undef;
   }
   $count_dl++;
@@ -616 +628 @@
   my $curchan = '';
   my ($pid, $block, $line, $link, $title);
+  my $c = 0;
   foreach my $tag ($tree->look_down('_tag' => 'td', 'class' => 'venue'))
   {
+    $c++;
     next if ($curchan eq $tag->as_text()); # Ignore repeated Station name
     $curchan = $tag->as_text();
@@ -671 +685 @@
       else
       {
-        print "Parsing error: No pid found in block.\n";
-      }
-    }
-  }
+        bad_parse("No pid found in guide data block");
+      }
+    }
+  }
+  bad_parse("Missing data") unless ($c);
 }
 
@@ -712 +727 @@
 
   $block = $tree->find('h1');
-  return undef unless ($block);  # site is probably sending that block page
+  return bad_parse("Mising title") unless ($block);  # site is probably sending that block page
 
   $show->{'title'} = [[ strip_whitespace($block->as_text()), $lang ]];
@@ -721 +736 @@
   }
 
-  $block = $tree->find('h3') or return undef;
+  $block = $tree->find('h3') or return bad_parse("Missing section");
   @rows = $block->look_down('_tag' => 'div');
   $date = $rows[1]->as_text();
@@ -732 +747 @@
   else
   {
-    return undef;  # Failed to parse
+    return bad_parse("Missing times");  # Failed to parse
   }
   if (Date_Cmp($show->{'start'}, $show->{'stop'}) == 1)
@@ -741 +756 @@
   $show->{'channel'} = $chanid{lc($rows[0]->find('span')->as_text())};
 
-  $block = $tree->find('_tag' => 'hr', 'noshade') or return undef;
+  $block = $tree->find('_tag' => 'hr', 'noshade') or return bad_parse("Missing subsection");
   $block = $block->right();
   if ($block->as_text())
@@ -858 +873 @@
 }
 
+sub bad_parse
+{
+  my $msg = shift;
+  print "Parsing error: $msg.\n";
+  $count_bad_parse++;
+  if ($count_bad_parse > 4 and !scalar(keys %shows))
+  {
+      print "ERROR! Unable to parse any shows! Looks like a major problem. " .
+            "Exiting.\n";
+      print stats(1);
+      exit 1;
+  }
+  return 0;
+}
+
 sub translate_category
 {

status

@@ -1 +1 @@
 application     shepherd            0.4.9
 grabber         yahoo7widget        1.60
-grabber         rex                 3.3.7
+grabber         rex                 3.3.8
 grabber         abc_website         2.03
 grabber         abc2_website        2.03