Changeset 45

Timestamp:

10/08/06 19:26:00 (7 years ago)

Author:

lincoln

Message:

abc_website.conf and abc2_website.conf files

Files:

• grabbers/abc2_website (modified) (6 diffs)
• grabbers/abc2_website.conf (added)
• grabbers/abc_website (modified) (6 diffs)
• grabbers/abc_website.conf (added)
• status (modified) (1 diff)

grabbers/abc2_website

@@ -12 +12 @@
 #    1.51  02oct06      --ready option
 #    1.52  03oct06      split out abc grabber into its own grabber
+#    1.55  09oct06      formalize --cheap option
 
 use strict;
 
 my $progname = "abc2_website";
-my $version = "1.54_03oct06";
+my $version = "1.55_00oct06";
 
 use LWP::UserAgent;
@@ -62 +63 @@
 my $opt_channels_file=  "";
 my $opt_no_cache =      0;
+my $opt_cheap =         0;
 my $opt_fast =          0;
 my $opt_warper =        0;
@@ -87 +89 @@
         'fast'          => \$opt_fast,
         'no-cache'      => \$opt_no_cache,
+        'cheap'         => \$opt_cheap,
         'debug+'        => \$debug,
         'warper'        => \$opt_warper,
@@ -120 +123 @@
 $starttime += (86400 * $opt_offset);
 
-&log(sprintf "going to grab %s%d%s days%s of data into %s (%s%s%s)",
+&log(sprintf "going to %s %s%d%s days%s of data into %s (%s%s%s)",
+        ($opt_cheap ? "verify (cache-validate)" : "grab"),
         ($opt_no_extra_days ? "" : "somewhere between "),
         $opt_days,
@@ -160 +164 @@
         --config-file=file      (ignored - historically used by grabbers not not this one)
         --no-cache              don't use a cache to optimize (reduce) number of web queries
+        --cheap                 validate contents of cache - fetch summary only, not details
         --cache-file=file       where to store cache (default "$opt_cache_file")
         --fast                  don't run slow - get data as quick as you can - not recommended
@@ -306 +311 @@
                         &add_cached_data($channel,$starttime,$cache_key);
                 } else {
-                        &get_one_abc_event($channel, $cache_key, $unprocessed_url[$i], "$xmlid detail pages ($i of $unprocessed_programmes)");
+                        &get_one_abc_event($channel, $cache_key, $unprocessed_url[$i], "$xmlid detail pages ($i of $unprocessed_programmes)") if (!$opt_cheap);
                 }
         }

grabbers/abc_website

@@ -12 +12 @@
 #    1.51  02oct06      --ready option
 #    1.52  03oct06      split out abc grabber into its own grabber
+#    1.55  09oct06      formalize --cheap option
 
 use strict;
 
 my $progname = "abc_website";
-my $version = "1.53_03oct06";
+my $version = "1.55_09oct06";
 
 use LWP::UserAgent;
@@ -62 +63 @@
 my $opt_channels_file=  "";
 my $opt_no_cache =      0;
+my $opt_cheap =         0;
 my $opt_fast =          0;
 my $opt_warper =        0;
@@ -87 +89 @@
         'fast'          => \$opt_fast,
         'no-cache'      => \$opt_no_cache,
+        'cheap'         => \$opt_cheap,
         'debug+'        => \$debug,
         'warper'        => \$opt_warper,
@@ -120 +123 @@
 $starttime += (86400 * $opt_offset);
 
-&log(sprintf "going to grab %s%d%s days%s of data into %s (%s%s%s)",
+&log(sprintf "going to %s %s%d%s days%s of data into %s (%s%s%s)",
+        ($opt_cheap ? "verify (cache-validate)" : "grab"),
         ($opt_no_extra_days ? "" : "somewhere between "),
         $opt_days,
@@ -160 +164 @@
         --config-file=file      (ignored - historically used by grabbers not not this one)
         --no-cache              don't use a cache to optimize (reduce) number of web queries
+        --cheap                 validate contents of cache - fetch summary only, not details
         --cache-file=file       where to store cache (default "$opt_cache_file")
         --fast                  don't run slow - get data as quick as you can - not recommended
@@ -304 +309 @@
                         &add_cached_data($channel,$starttime,$cache_key);
                 } else {
-                        &get_one_abc_event($channel, $cache_key, $unprocessed_url[$i], "$xmlid detail pages ($i of $unprocessed_programmes)");
+                        &get_one_abc_event($channel, $cache_key, $unprocessed_url[$i], "$xmlid detail pages ($i of $unprocessed_programmes)") if (!$opt_cheap);
                 }
         }

status

@@ -2 +2 @@
 rex:3.0.1:grabber
 oztivo:0.4:grabber
-abc_website:1.53:grabber
-abc2_website:1.54:grabber
+abc_website:1.55:grabber
+abc2_website:1.55:grabber
 yahoo7widget:1.53:grabber
 jrobbo:0.01:grabber