Changeset 506

Timestamp:

02/10/07 16:26:37 (6 years ago)

Author:

lincoln

Message:

microgap support for sbsnews

Files:

• grabbers/sbsnews_website (modified) (11 diffs)
• grabbers/sbsnews_website.conf (modified) (1 diff)
• status (modified) (1 diff)

grabbers/sbsnews_website

@@ -15 +15 @@
 
 my $progname = "sbsnews_website";
-my $version = "0.05";
+my $version = "0.06";
 
 use LWP::UserAgent;
@@ -30 +30 @@
 my $script_start_time = time;
 my %stats;
-my $channels, my $opt_channels;
+my $channels, my $opt_channels, my $gaps;
 my $tv_guide;
 
@@ -58 +58 @@
         'offset=i'      => \$opt->{offset},
         'channels_file=s' => \$opt->{channels_file},
+        'gaps_file=s'   => \$opt->{gaps_file},
         'output=s'      => \$opt->{outputfile},
         'fast'          => \$opt->{fast},
@@ -88 +89 @@
 $starttime += (86400 * $opt->{offset});
 
-&log(sprintf "going to fetch %d days%s of data into %s (%s)",
+&log(sprintf "going to %sfetch %d days%s of data into %s (%s)",
+        (defined $opt->{gaps_file} ? "micro-gap " : ""),
         $opt->{days},
         ($opt->{offset} ? " (skipping first $opt->{offset} days)" : ""),
@@ -99 +101 @@
         no warnings 'all'; eval <>; die "$@" if $@;
 } else {
-        die "WARNING: channels file $opt->{channels_file} could not be read\n";
+        die "WARNING: channels file $opt->{channels_file} could not be read: $!\n";
 }
 
@@ -106 +108 @@
 die "no SBSNEWS channel found in channel lineup from $opt->{channels_file}\n"
   unless (defined $channels->{'SBS News'});
+
+if (defined $opt->{gaps_file}) {
+        if (-r $opt->{gaps_file}) {
+                local (@ARGV, $/) = ($opt->{gaps_file});
+                no warnings 'all'; eval <>; die "$@" if $@;
+        } else {
+                die "WARNING: gaps file $opt->{gaps_file} could not be read: $!\n";
+        }
+
+        die "no SBSNEWS channel in gaps lineup, nothing to do!\n"
+          unless ((defined $gaps) && (defined $gaps->{'SBS News'}));
+}
 
 &get_sbsnews_data($starttime,$endtime);
@@ -130 +144 @@
         --lang=[s]              set language of xmltv output data (default $opt->{lang})
         --channels_file=file    where to get channel data from
+        --gaps_file=file        micro-fetch gaps only
+
 EOF
 ;
@@ -291 +307 @@
                                                 next if ($progname eq "");
 
-                                                my $progstart = strftime "%Y%m%d%H%M",localtime(($start_time + $day_range[($column_is_day[$col])]));
-                                                my $progstop = strftime "%Y%m%d%H%M",localtime(($stop_time + $day_range[($column_is_day[$col])]));
-
-                                                &log((sprintf "got prog col %d: start=%d (%s), stop=%d (%s), '%s', day=%d",
-                                                  $col,$start_time,$progstart,$stop_time,$progstop,$progname,$column_is_day[$col]))
-                                                  if $opt->{debug};
-
                                                 my $prog;
                                                 $prog->{channel} = $channels->{'SBS News'};
                                                 $prog->{title} = [[ $progname, $opt->{lang} ]];
-                                                $prog->{start} = $progstart;
-                                                $prog->{stop} = $progstop;
+                                                $prog->{starttime} = $start_time + $day_range[($column_is_day[$col])];
+                                                $prog->{stoptime} = $stop_time + $day_range[($column_is_day[$col])];
+
                                                 $prog->{category} = [[ 'News', undef ]] if ($progname =~ /News/i);
                                                 $prog->{daynum} = ($day_range[($column_is_day[$col])] - $currtime) / (60*60*24);
 
-                                                $tv_guide->{$progstart} = $prog;
+                                                $tv_guide->{($prog->{starttime})} = $prog;
+
+                                                &log((sprintf "got prog col %d: start=%d (%s), stop=%d (%s), '%s', day=%d",
+                                                  $col,$start_time,$prog->{starttime},$stop_time,$prog->{stoptime},$progname,$column_is_day[$col]))
+                                                  if $opt->{debug};
+
                                                 $stats{seen_progs}++;
                                         }
@@ -350 +365 @@
         my ($raw, $page, $base);
 
-        $retrycount = 5 if ($retrycount == 0);
+        $retrycount = 8 if ($retrycount == 0);
         my $request = HTTP::Request->new(GET => $url);
         $request->header('Accept-Encoding' => 'gzip');
@@ -362 +377 @@
 
                 $stats{http_failed_requests}++;
-                $stats{slept_for} += 10;
-                sleep 10;
+                $stats{slept_for} += 20;
+                sleep 20;
         }
         if (!($response->is_success)) {
@@ -449 +464 @@
                 if (($tv_guide->{$prog}->{daynum} >= $opt->{offset}) &&
                     ($tv_guide->{$prog}->{daynum} < $opt->{days})) {
+
+                        # if we are fetching microgaps, skip if this isn't
+                        # in a micro-gap
+                        if (defined $opt->{gaps_file}) {
+                                my $found_gap_match = 0;
+                                foreach my $g (@{($gaps->{'SBS News'})}) {
+                                        my ($s, $e) = split(/-/,$g);
+
+                                        $found_gap_match = 1 if
+                                           ((($s >= $tv_guide->{$prog}->{starttime}) &&
+                                             ($s <= $tv_guide->{$prog}->{stoptime})) ||
+                                            (($e >= $tv_guide->{$prog}->{starttime}) &&
+                                             ($e <= $tv_guide->{$prog}->{stoptime})) ||
+                                            (($s <= $tv_guide->{$prog}->{starttime}) &&
+                                             ($e >= $tv_guide->{$prog}->{stoptime})));
+                                }
+                                if (!$found_gap_match) {
+                                        $stats{gaps_skipped}++;
+                                        next;
+                                } else {
+                                        $stats{gaps_included}++;
+                                }
+                        }
+
+                        $tv_guide->{$prog}->{start} = POSIX::strftime("%Y%m%d%H%M",localtime($tv_guide->{$prog}->{starttime}));
+                        $tv_guide->{$prog}->{stop} = POSIX::strftime("%Y%m%d%H%M",localtime($tv_guide->{$prog}->{stoptime}));
                         delete $tv_guide->{$prog}->{daynum};
+                        delete $tv_guide->{$prog}->{starttime};
+                        delete $tv_guide->{$prog}->{stoptime};
                         &cleanup($tv_guide->{$prog});
 

grabbers/sbsnews_website.conf

@@ -9 +9 @@
             'max_days' => 7,
             'category' => 2,
+            'micrograbs' => 1,
             'quality' => 3,
             'max_runtime' => '2',

status

@@ -4 +4 @@
 grabber         abc_website         2.07
 grabber         abc2_website        2.07
-grabber         sbsnews_website     0.05
+grabber         sbsnews_website     0.06
 grabber         oztivo              0.14
 grabber         jrobbo              0.06