Changeset 50

Timestamp:

10/08/06 22:29:27 (7 years ago)

Author:

max

Message:

Improvements to Rex, incl. gzip compression support.

Files:

• grabbers/rex (modified) (13 diffs)
• status (modified) (1 diff)

grabbers/rex

@@ -3 +3 @@
 # "Rex"
 
-my $version  = '3.0.1';
+my $version  = '3.2.1';
 
 # An Australian TV Guide Grabber (a.k.a. tv_grab_au)
@@ -55 +55 @@
 #                    --rebuild-cache options; exit on unknown option
 # 3.0.0   : Shepherd compatibility
-# 3.0.1   : Added --ready option
+# 3.1.0   : Feature: --ready option
+# 3.2.0   : Feature: gzip compression, report KB downloaded
+# 3.2.1   : Bugfix : handle failed downloads better
 
 use strict;
@@ -83 +85 @@
 my $cache_file = "$output_dir/cache.dat";
 
-my ($count_dl, $count_detail, $count_bad, $count_cache, $count_changes) = (0) x 5;
+my ($count_dl, $count_detail, $count_bad, $count_cache, $count_changes, $count_kb) = (0) x 6;
 
 my $DATASOURCE             = "http://www.yourtv.com.au";
@@ -106 +108 @@
 my %shows;
 my $numshows;
+my $dcount;
 
 # ---------------------------------------------------------------------------
@@ -231 +234 @@
   $firstfetch = time();
   my $show;
-  my $dcount = 0;
   foreach my $pid (keys %$precache)
   {
@@ -251 +253 @@
     unless ($shows{$pid})
     {
-      $dcount++;
-      refresh_ua() if ($dcount % 20 == 0); # don't wait for error page
       $show = download_show($pid);
       if ($show)
@@ -261 +261 @@
       else
       {
-        $count_bad++;
         print "Failed to parse show $pid.\n";
       }
-      sleep int(rand(10));
+      sleep int(2 + rand(5));
     }
     if ($opt->{stats} and time() - $laststats >= $opt->{stats})
@@ -323 +322 @@
   $recurse_count ||= 0;
   return undef if ($recurse_count > 2);
-  
+ 
+  $dcount++;
+  refresh_ua() if ($dcount % 20 == 0); # don't wait for error page
+
+  my $result;
+
   print "Downloading # $pid.\n" if ($debug);
   my $detailsdata = get_page($DATASOURCE_DETAIL .
                       '?action=session_info&event_id=' . $pid .
                       '&sid=' . $sid . '&loc=grid');
-  unless ($detailsdata)
+  $result = parse_details($detailsdata) if ($detailsdata);
+  unless ($detailsdata and $result)
   {
     print "Download failed.\n" if ($debug);
+    sleep(5);
     $count_bad++;
     refresh_ua();
@@ -336 +342 @@
   }
 
-  return parse_details($detailsdata);
+  return $result;
 }
 
@@ -418 +424 @@
         1 );
   $ua->cookie_jar()->scan(\&refresh_sid);
+
+  $dcount = 0;
 }
 
@@ -483 +491 @@
   $ret .= sprintf(
         " %d shows grabbed\n" .
-        " %d downloads, including %d detail pages\n" .
+        " %d downloads, including %d detail pages (%d KB)\n" .
         " %d cache hits, %d changes from cache, %d failed downloads\n",
         scalar(keys %shows),
-        $count_dl, $count_detail, $count_cache, $count_changes, $count_bad);
+        $count_dl, $count_detail, $count_kb, 
+        $count_cache, $count_changes, $count_bad);
   $ret .= " Time elapsed: " . timestats($t) . "\n";
   unless ($finished or !$count_detail)
@@ -532 +541 @@
   
   $request->uri() =~ s/^http:\/\//$WW/ if $opt->{warper};
+
+  $request->header('Accept-Encoding' => 'gzip');
 
   print "Fetching: " . $request->as_string() . "\n" if ($debug);
@@ -552 +563 @@
   $count_dl++;
   my $page = $response->content();
+  $count_kb += (do {use bytes; length($page)}) / 1024;
+
+  if ($response->header('Content-Encoding')
+      and
+      $response->header('Content-Encoding') eq 'gzip') 
+  {
+      $page = Compress::Zlib::memGunzip($response->content());
+  }
+
   $page =~ s/ / /g;
   return $page;

status

@@ -1 +1 @@
 shepherd:0.2.8:shepherd
-rex:3.0.1:grabber
+rex:3.2.1:grabber
 oztivo:0.4:grabber
 abc_website:1.55:grabber