Changeset 555 for grabbers/yahoo7web

Timestamp:

02/25/07 14:59:27 (6 years ago)

Author:

lincoln

Message:

yahoo7web: 1. harden use of Storable a bit, 2. cache day/summary pages for up to 4 hours so as to make micrograbbing more efficient

Files:

• grabbers/yahoo7web (modified) (10 diffs)

grabbers/yahoo7web

@@ -8 +8 @@
 
 my $progname = "yahoo7web";
-my $version = "0.11";
+my $version = "0.12";
 
 use LWP::UserAgent;
@@ -166 +166 @@
 sub read_cache
 {
-        # convert old-style Data::Dumper cache to new format
-        if (-r "yahoo7web.cache") {
-                local (@ARGV, $/) = ("yahoo7web.cache");
-                no warnings 'all'; eval <>; die "$@" if $@;
-
-                &write_cache;
-                unlink "yahoo7web.cache";
-        }
-
         if (-r $opt->{cache_file}) {
-                my $store = Storable::retrieve($opt->{cache_file});
-                $data_cache = $store->{data_cache};
+                my $store;
+                eval { $store = Storable::retrieve($opt->{cache_file}); };
+                $data_cache->{progs} = $store->{data_cache} if (defined $store->{data_cache});
+
+                if (defined $store->{day_cache}) {
+                        $data_cache->{day} = $store->{day_cache};
+
+                        # age day cache on reading..
+                        for my $url (keys %{($data_cache->{day})}) {
+                                if ($data_cache->{day}->{$url}->{fetched} < (time-(4*3600))) {
+                                        delete $data_cache->{day}->{$url};
+                                        $stats{expired_url_from_cache}++;
+                                }
+                        }
+                }
         } else {
                 printf "WARNING: no programme cache $opt->{cache_file} - have to fetch all details\n";
@@ -191 +195 @@
 sub write_cache
 {
-        # cleanup old entries from cache
-        for my $cache_key (keys %{$data_cache}) {
-                my ($starttime, @rest) = split(/:/,$cache_key);
-                if ($starttime < (time-86400)) {
-                        delete $data_cache->{$cache_key};
-                        $stats{expired_from_cache}++;
+        # cleanup old prog entries from cache
+        if (defined $data_cache->{progs}) {
+                for my $cache_key (keys %{($data_cache->{progs})}) {
+                        my ($starttime, @rest) = split(/:/,$cache_key);
+                        if ($starttime < (time-86400)) {
+                                delete $data_cache->{progs}->{$cache_key};
+                                $stats{expired_from_cache}++;
+                        }
                 }
         }
 
         my $store;
-        $store->{data_cache} = $data_cache;
+        $store->{data_cache} = $data_cache->{progs} if (defined $data_cache->{progs});
+        $store->{day_cache} = $data_cache->{day} if (defined $data_cache->{day});
 
         Storable::store($store, $opt->{cache_file});
@@ -485 +492 @@
 
                         my $url = sprintf "http://au.tv.yahoo.com/tv-guide/?hour=%s&min=%s&date=%s&mon=%s&year=%s&tvrg=%s&next=%s",
-                                POSIX::strftime("%H",localtime($starttime)),
-                                POSIX::strftime("%M",localtime($starttime)),
-                                POSIX::strftime("%d",localtime($starttime)),
-                                POSIX::strftime("%m",localtime($starttime)),
-                                POSIX::strftime("%Y",localtime($starttime)),
+                                POSIX::strftime("%H",localtime($starttime_midnight)),
+                                POSIX::strftime("%M",localtime($starttime_midnight)),
+                                POSIX::strftime("%d",localtime($starttime_midnight)),
+                                POSIX::strftime("%m",localtime($starttime_midnight)),
+                                POSIX::strftime("%Y",localtime($starttime_midnight)),
                                 $opt->{region}, $currtime;
 
@@ -515 +522 @@
         my ($url, $day_num, $day_start) = @_;
         my %chan_map = &build_channel_quirks_map;
-
-        my $data = &get_url($url,5);
-        return if (!$data);
+        my $data;
+
+        if ((defined $data_cache->{day}->{$url}) &&
+            (defined $data_cache->{day}->{$url}->{data})) {
+                $data = $data_cache->{day}->{$url}->{data};
+                $stats{used_cached_day_page}++;
+        } else {
+                $data = &get_url($url,5);
+                return if (!$data);
+                $data_cache->{day}->{$url}->{fetched} = time;
+                $data_cache->{day}->{$url}->{data} = $data;
+        }
 
         my $tree = HTML::TreeBuilder->new_from_content($data);
@@ -635 +651 @@
                 $prog_count++;
 
-                if ((!defined $data_cache->{$cache_key}) &&
+                if ((!defined $data_cache->{progs}->{$cache_key}) &&
                     (!defined $opt->{no_details}) &&
                     (defined $prog->{event_id}) &&
@@ -650 +666 @@
 
                 # if we got additional details from the cache, add them now
-                if (defined $data_cache->{$cache_key}) {
-                        foreach my $key (keys %{($data_cache->{$cache_key})}) {
-                                $prog->{$key} = $data_cache->{$cache_key}->{$key};
+                if (defined $data_cache->{progs}->{$cache_key}) {
+                        foreach my $key (keys %{($data_cache->{progs}->{$cache_key})}) {
+                                $prog->{$key} = $data_cache->{progs}->{$cache_key}->{$key};
                         }
                 }
@@ -708 +724 @@
                                 ; # nothing
                         } elsif ($f eq "subtitle") {
-                                $data_cache->{$cache_key}->{'sub-title'} = [[ $v, $opt->{lang} ]];
+                                $data_cache->{progs}->{$cache_key}->{'sub-title'} = [[ $v, $opt->{lang} ]];
                         } elsif ($f eq "description") {
-                                $data_cache->{$cache_key}->{desc} = [[ $v, $opt->{lang} ]];
+                                $data_cache->{progs}->{$cache_key}->{desc} = [[ $v, $opt->{lang} ]];
                         } elsif ($f eq "genre") {
                                 push(@genre, translate_category($v));
                         } elsif ($f eq "captions") {
-                                $data_cache->{$cache_key}->{subtitles} = [ { 'type' => 'teletext' } ] if ($v eq "true");
+                                $data_cache->{progs}->{$cache_key}->{subtitles} = [ { 'type' => 'teletext' } ] if ($v eq "true");
                         } elsif ($f eq "start_date") {
                                 ; # nothing
@@ -740 +756 @@
                         }
 
-                        $data_cache->{$cache_key}->{category} = [[ @genre ]] if ($#genre != -1);
+                        $data_cache->{progs}->{$cache_key}->{category} = [[ @genre ]] if ($#genre != -1);
 
                 } else {
@@ -748 +764 @@
         }
 
-        printf "DEBUG: cached entries for '$cache_key': ".Dumper($data_cache->{$cache_key})
+        printf "DEBUG: cached entries for '$cache_key': ".Dumper($data_cache->{progs}->{$cache_key})
           if (defined $opt->{debug});
 }