Changeset 524 for grabbers/yahoo7web

Timestamp:

02/15/07 00:34:55 (6 years ago)

Author:

lincoln

Message:

optimize yahoo7web: when fetching microgaps, only fetch summary pages for hours that have gaps

Files:

• grabbers/yahoo7web (modified) (4 diffs)

grabbers/yahoo7web

@@ -8 +8 @@
 
 my $progname = "yahoo7web";
-my $version = "0.09";
+my $version = "0.10";
 
 use LWP::UserAgent;
@@ -480 +480 @@
                         next if (($currtime + (3 * 60 * 60)) < $starttime); # no point fetching the past
 
+                        # if we are fetching microgaps, skip this summary page if we aren't
+                        # interested in anything from it anyway
+                        next if ((defined $opt->{gaps_file}) && (!window_is_within_microgap($currtime,$currtime+(60*60*3))));
+
                         my $url = sprintf "http://au.tv.yahoo.com/tv-guide/?hour=%s&min=%s&date=%s&mon=%s&year=%s&tvrg=%s&next=%s",
                                 POSIX::strftime("%H",localtime($starttime)),
@@ -588 +592 @@
                                         $progs_in_table++;
 
-                                        # if we are fetching microgaps, skip if this isn't
-                                        # in a micro-gap.
+                                        # if we are fetching microgaps, skip if this isn't in a micro-gap.
                                         if (defined $opt->{gaps_file}) {
-                                                my $found_gap_match = 0;
-                                                if (defined $gaps->{$prog->{channel}}) {
-                                                        foreach my $g (@{($gaps->{$prog->{channel}})}) {
-                                                                my ($s, $e) = split(/-/,$g);
-                                                                $found_gap_match = 1 if
-                                                                  ((($s >= $prog->{starttime}) && ($s <= $prog->{stoptime})) ||
-                                                                   (($e >= $prog->{starttime}) && ($e <= $prog->{stoptime})) ||
-                                                                   (($s <= $prog->{starttime}) && ($e >= $prog->{stoptime})));
-                                                        }
-                                                }
-                                                if (!$found_gap_match) {
-                                                        $stats{gaps_skipped}++;
-                                                        next;
-                                                } else {
-                                                        $stats{gaps_included}++;
-                                                }
+                                                next if (!window_is_within_microgap($prog->{starttime},$prog->{stoptime},$this_chan));
+                                                $stats{gaps_included}++;
                                         }
 
@@ -859 +848 @@
 ##############################################################################
 
+sub window_is_within_microgap
+{
+        my ($start, $stop, $channel) = @_;
+
+        return window_channel_is_within_microgap($start, $stop, $channel) if (defined $channel);
+
+        foreach my $ch (keys %{$channels}) {
+                return 1 if window_channel_is_within_microgap($start, $stop, $ch);
+        }
+        return 0;
+}
+
+sub window_channel_is_within_microgap
+{
+        my ($start, $stop, $channel) = @_;
+
+        if (defined $gaps->{$channel}) {
+                foreach my $g (@{($gaps->{$channel})}) {
+                        my ($s, $e) = split(/-/,$g);
+                        return 1 if
+                          ((($s >= $start) && ($s <= $stop)) ||
+                           (($e >= $start) && ($e <= $stop)) ||
+                           (($s <= $start) && ($e >= $stop)));
+                }
+        }
+        $stats{gaps_skipped}++;
+        return 0;
+}
+
+##############################################################################
+