Changeset 481

Timestamp:

02/01/07 15:29:27 (6 years ago)

Author:

max

Message:

Micrograb support for Rex

Files:

• grabbers/rex (modified) (16 diffs)
• grabbers/rex.conf (modified) (1 diff)
• status (modified) (1 diff)

grabbers/rex

@@ -3 +3 @@
 # "Rex"
 
-my $version  = '3.3.9';
+my $version  = '3.4.0';
 
 # An Australian TV Guide Grabber (a.k.a. tv_grab_au)
@@ -71 +71 @@
 # 3.3.8   : Bugfix : better handling of failed downloads & parses
 # 3.3.9   : Bugfix : more descriptive dies
+# 3.4.0   : Feature: micrograbbing (--gaps_file)
 
 use strict;
@@ -83 +84 @@
 use JavaScript;
 use Storable;
-use Cwd;
+use POSIX;
 
 use XMLTV;
@@ -110 +111 @@
 my $channels = {};
 my $opt_channels = {};
+my $gaps;
 my %chanid;
 my $cached;
+my $lastpids;
 my $precache;
 my $ua;
@@ -140 +143 @@
 
 read_channels_file();
+read_gaps_file();
 
 if ($opt->{test})
@@ -407 +411 @@
   if ($ua)
   {
+     print stats() if ($debug and $opt->{stats});
      print "Sleeping...\n" if ($debug);
      sleep 5 + int(rand(20));
@@ -652 +657 @@
           if ($start)
           {
-            addme($pid, $title, $start, $pextra);
+            addme($pid, $title, $start, $pextra, $curchan);
             $pextra = undef;
             $title = undef;
@@ -682 +687 @@
       if ($pid)
       {
-        addme($pid, $title, $start, $pextra);
+        addme($pid, $title, $start, $pextra, $curchan);
       }
       else
@@ -695 +700 @@
 sub addme
 {
-  my ($pid, $title, $start, $pextra) = @_;
+  my ($pid, $title, $start, $pextra, $curchan) = @_;
   
   if ($precache->{$pid})
@@ -714 +719 @@
                           'sub-title' => $pextra
                         };
+    my $lastpid = $lastpids->{$curchan};
+    if ($lastpid)
+    {
+        $precache->{$lastpid}->{stop} = $start;
+        if ($start < $precache->{$lastpid}->{start}) 
+        {
+            die "Logical error in stop time processing: pid $pid";
+        }
+        if ($gaps)
+        {
+            if (is_outside_gaps($curchan, $precache->{$lastpid}->{start}, $precache->{$lastpid}->{stop}))
+            {
+                delete $precache->{$lastpid};
+            }
+        }
+    }
+    $lastpids->{$curchan} = $pid;
   }
 }
@@ -903 +925 @@
 }
 
+sub is_outside_gaps
+{
+  my ($ch, $start, $stop) = @_;
+
+  $start = udate_to_epoch($start);
+  $stop = udate_to_epoch($stop);
+
+  print "Gap check: $start to $stop vs:\n";
+  foreach my $gap (@{$gaps->{$ch}})
+  {
+      if ($gap =~ /(\d+)-(\d+)/)
+      {
+          print " - $1 - $2\n";
+          return 0 if ($stop > $1 and $start < $2);
+      }
+  }
+  return 1;
+}
+
+sub udate_to_epoch
+{
+    my $udate = shift;
+
+    if ($udate =~ /(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)/)
+    {
+        my ($year, $month, $day, $hour, $minute, $second) = ($1, $2, $3, $4, $5, $6);
+        $month--;
+        $year -= 1900;
+        return POSIX::strftime("%s", $second, $minute, $hour, $day, $month, $year);
+    }
+    die "Unable to parse udate $udate!";
+}
+
 sub readjs {
   my $data = shift;
@@ -915 +970 @@
  
 sub read_channels_file 
-{   
-    print "Reading channels file: $opt->{channels_file}\n";
-    if (-r $opt->{channels_file})
-    {
-        local (@ARGV, $/) = ($opt->{channels_file});
+{
+    read_config_file('channels', 'channels_file');
+}
+
+sub read_gaps_file
+{
+    read_config_file('gaps', 'gaps_file');
+#    foreach my $ch (keys %$gaps)
+#    {
+#       foreach my $gap (@{$gaps->{$ch}})
+#       {
+#           if ($gap =~ /(\d+)-(\d+)/)
+#           {
+#               my ($start, $end) = ($1, $2);
+#               $gap = epoch
+
+
+}
+
+sub read_config_file
+{
+    my ($name, $arg) = @_;
+
+    return unless ($opt->{$arg});
+    print "Reading $name file: $opt->{$arg}\n";
+    if (-r $opt->{$arg})
+    {
+        local (@ARGV, $/) = ($opt->{$arg});
         no warnings 'all';
         eval <>;
-        die "Can't parse channels file: $@" if ($@);
+        die "Can't parse $name file: $@" if ($@);
     }
     else
     {
-        print "Unable to read config file.\n";
+        print "Unable to read $name file.\n";
     }
 }
@@ -948 +1026 @@
                         config-file=s
                         channels_file=s
+                        gaps_file=s
                         stats=i
                         test
@@ -980 +1059 @@
           "Output file: " . ($opt->{output} ? $opt->{output} : "None") . "\n" .
           "Chann file : $opt->{channels_file}\n" .
+          ($opt->{gaps_file} ? "Gaps file  : " . $opt->{gaps_file} . "\n" : '') .
           "Statistics : " . ($opt->{stats} ? "every " . $opt->{stats} . " seconds" : "off") . "\n";
   }
@@ -1014 +1094 @@
           'region' => 94,
           'stats' => 300,
-          'output' => cwd() . '/output.xmltv',
-          'cache-file' => cwd() . '/cache.dat',
-          'channels_file' => cwd() . '/channels.conf'
+          'output' => getcwd() . '/output.xmltv',
+          'cache-file' => getcwd() . '/cache.dat',
+          'channels_file' => getcwd() . '/channels.conf'
   };
                                               
@@ -1066 +1146 @@
     --output <file>        Write XML into the specified file
     --channels_file <file> Read channel subscriptions from file
+    --gaps_file <file>     Read gaps from file
 
     --region <n>           Grab data for region code <n>

grabbers/rex.conf

@@ -15 +15 @@
                 },
             'max_runtime' => '240',
-            'option_days' => '--days'
+            'option_days' => '--days',
+            'micrograbs' => 1
           };

status

@@ -1 +1 @@
 application     shepherd            0.4.39
 grabber         yahoo7widget        1.77
-grabber         rex                 3.3.9-r1
+grabber         rex                 3.4.0
 grabber         abc_website         2.05
 grabber         abc2_website        2.05