Changeset 3

Timestamp:

10/03/06 23:55:08 (7 years ago)

Author:

max

Message:

Lincoln's first pass at XML analysis.

Files:

• shepherd (modified) (15 diffs)
• status (modified) (1 diff)

shepherd

@@ -3 +3 @@
 # "Shepherd"
 
-my $version = '0.2.3';
+my $version = '0.2.5ng';
 
 # A wrapper for various Aussie TV guide data grabbers
@@ -19 +19 @@
 # 0.2.2   : --check
 # 0.2.3   : Bugfix: archives correctly
+# 0.2.5ng : multi-grabber (potentially with partial data) & postprocessor support
 #
 # ToDo:
@@ -24 +25 @@
 #   than whuffy. Store this in shepherd.conf. Whuffy is merely the first
 #   mirror.
-# * parse that XMLTV output and do something smart with it!
 # * Make it check compilation after installing by calling --version or
 #   --desc or --ready
@@ -62 +62 @@
 my $debug = 1;
 my $grabbers = { };
+my $preferred; # obsolete but may still exist in shepherd.conf
 my $region;
 my $channels;
-my $preferred;
 my $config_file =   "$CWD/$progname.conf";
 my $channels_file = "$CWD/channels.conf";
 my $days;
+my $grabber_data = { };
+my $master_starttime = time;
 
 # ---------------------------------------------------------------------------
@@ -113 +115 @@
 }
 
-if ($opt->{prefer})
-{
-    set_preferred($opt->{prefer});
-}
+&set_order($opt->{setorder}) if ($opt->{setorder});
 
 if ($opt->{check})
@@ -123 +122 @@
 }
 
-if ($opt->{enable} or $opt->{disable} or $opt->{prefer} or $opt->{check})
+if ($opt->{enable} or $opt->{disable} or $opt->{setorder} or $opt->{check})
 {
     write_config_file();
@@ -168 +167 @@
 sub grab_data
 {
-    my $grabber = choose_grabber();
-
-    unless ($grabber)
+    my $used_grabbers = 0;
+    my $need_more_data = 1;
+
+    # iterate across grabbers until we have all our data we want (or need)
+    foreach my $grabber (sort { $grabbers->{$a}->{order} <=> $grabbers->{$b}->{order} } keys %$grabbers) {
+        next if ($grabbers->{$grabber}->{disabled});
+        $used_grabbers++;
+
+        print "\nUsing grabber: $grabber.\n\n";
+
+        my $output = $opt->{output};
+        $output ||= "$CWD/output.xmltv";
+        my $comm = "$GRABBER_DIR/$grabber/$grabber " .
+                   "--region $region " .
+                   "--channels_file $channels_file " .
+                   "--output $output";
+
+        # NOTE: ideally a grabber could be instructed to fetch partial data through --channel, --starttime & --endtime
+        # we don't have that for now so instead whenever there is missing data, ALL 7 days for all channels will be collected
+        # FIXME FUTURE: call grabbers just with what we want...
+        $comm .= " --days $days" if ($days);
+        $comm .= " --offset $opt->{offset}" if ($opt->{offset});
+        $comm .= " --debug" if ($debug);
+        $comm .= " @ARGV" if (@ARGV);
+        print "Excuting command:\n$comm\n\n";
+
+        chdir "$GRABBER_DIR/$grabber/";
+        system($comm);
+        chdir $CWD;
+
+        # soak up the data we just collected
+        &soak_up_grabber_data($grabber, $output);
+
+        # check to see if we have all the data we want
+        $need_more_data = &analyze_grabber_data();
+
+        last if (!$need_more_data);
+    }
+
+
+    if ($used_grabbers == 0)
     {
         print "No valid grabbers installed/enabled!\n";
         return;
     }
-    print "\nUsing grabber: $grabber.\n\n";
-
-    my $output = $opt->{output};
-    $output ||= "$CWD/output.xmltv";
-    my $comm = "$GRABBER_DIR/$grabber/$grabber " .
-               "--region $region " .
-               "--channels_file $channels_file " .
-               "--output $output";
-    $comm .= " --days $days" if ($days);
-    $comm .= " --offset $opt->{offset}" if ($opt->{offset});
-    $comm .= " --debug" if ($debug);
-    $comm .= " @ARGV" if (@ARGV);
-    print "Excuting command:\n$comm\n\n";
-
-    chdir "$GRABBER_DIR/$grabber/";
-    system($comm);
-    chdir $CWD;
-}
-
-sub choose_grabber
-{
-    return $preferred if ($preferred 
-            and $grabbers->{$preferred} 
-            and !$grabbers->{$preferred}->{disabled});
-    foreach (keys %$grabbers)
-    {
-        return $_ unless ($grabbers->{$_}->{disabled});
-    }
-    return undef;
+
+    if ($need_more_data)
+    {
+        print "Ran through all grabbers but still missing data!!! :(\n";
+        return;
+    }
+}
+
+
+# interpret xmltv data from this grabber
+sub soak_up_grabber_data
+{
+    my ($grabber, $output) = @_;
+    eval { $grabber_data->{$grabber} = XMLTV::parsefiles($output); };
+}
+
+
+# analyze grabber data - do we have all the data we want?
+# returns 1 if we need more data, 0 if we have all we want
+sub analyze_grabber_data
+{
+    # normalize starttime to beginning of hour
+    my ($sec,$min,@rest) = localtime($master_starttime);
+    my $starttime = $master_starttime - ((60 * $min) + $sec);
+    my $endtime;
+
+    if ($days) {
+        $endtime = $starttime + ($days * 86400);
+    } else {
+        $endtime = $starttime + (7*86400);
+    }
+    $starttime += (86400 * $opt->{offset};
+
+    # XXX TODO!  for now just return 0 - we got all our data!
+    return 0;
+
+    # iterate across channels
+    #foreach my $channel (sort keys %{$channels}) {
+        # iterate across days
+
+    return 0;
 }
 
@@ -393 +442 @@
 }
 
-sub set_preferred
-{
-    my $proggy = shift;
-
-    if (!$grabbers->{$proggy})
-    {
-        print "No such grabber: \"$proggy\".\n";
-    }
-    else
-    {
-        $preferred = $proggy;
+sub set_order
+{
+    my $order = shift;
+    my $all_ok = 1;
+
+    # first check that user supplied a valid list of grabbers
+    if ($order) {
+        foreach my $proggy (split(/,/,$order)) {
+            next if !$proggy;
+            if (!$grabbers->{$proggy}) {
+                $all_ok = 0;
+                print "Invalid grabber: \"$proggy\".\n";
+            }
+        }
+    }
+    if ($all_ok) {
+        # if list was ok then first reset current order to zero
+        foreach my $proggy (keys %$grabbers) {
+            $grabbers->{$proggy}->{order} = 0;
+        }
+
+        # and now set order
+        my $order_num = 1;
+        if ($order) {
+            foreach my $proggy (split(/,/,$order)) {
+                $grabbers->{$proggy}->{order} = $order_num;
+                $order_num++;
+            }
+        }
+
+        # set order of any grabbers not specified in a random manner
+        foreach my $proggy (sort keys %$grabbers) {
+            if ($grabbers->{$proggy}->{order} == 0) {
+                $grabbers->{$proggy}->{order} = $order_num+int(rand(100));
+            }
+        }
+
+        # .. and finally normalize the order (& show the user the order we chose)
+        print "Grabber order set as follows:\n";
+        $order_num = 0;
+        $opt->{setorder} = "";
+        foreach my $proggy (sort { $grabbers->{$a}->{order} <=> $grabbers->{$b}->{order} } keys %$grabbers) {
+            $order_num++;
+            $grabbers->{$proggy}->{order} = $order_num;
+            $opt->{setorder} .= "$proggy" . ",";
+            printf " #%d. %s%s\n",$grabbers->{$proggy}->{order},$proggy,($grabbers->{$proggy}->{disabled} ? " [disabled]" : "");
+        }
     }
 }
@@ -429 +514 @@
 {
     read_file($config_file, 'configuration');
+
+    # if we are updating from a previous rev of shepherd.conf we may not
+    # have any 'order' fields set .. check here
+    my $found_order = 1;
+    foreach (keys %$grabbers)
+    {
+        $found_order = 0 if (!defined $grabbers->{$_}->{order});
+    }
+    if (($found_order == 0) && (!$opt->{setorder}))
+    {
+        # at least one 'order' was missing .. we need to put it in!
+        printf "Legacy shepherd.conf file didn't contain any grabber order! Automatically updating using a random order, use --setorder to manually set this if you care.\n";
+        &set_order();
+        $opt->{setorder} = "";
+    }
 }
 
@@ -466 +566 @@
     open(CONF, ">$config_file") or die "cannot write to $config_file: $!";
     print CONF Data::Dumper->Dump(
-        [$region,  $preferred,  $grabbers ],
-        ["region", "preferred", "grabbers" ]);
+        [$region,  $grabbers ],
+        ["region", "grabbers" ]);
     close CONF;
     print "\nUpdated configuration file $config_file.\n" if ($debug);
@@ -499 +599 @@
               'noupdate'        => \$opt->{noupdate},
 
-              'prefer=s'        => \$opt->{prefer},
               'disable=s'       => \$opt->{disable},
               'enable=s'        => \$opt->{enable},
+              'setorder=s'      => \$opt->{setorder},
 
               'days=i'          => \$days,
@@ -576 +676 @@
 
 
-    print "\nRandomly selecting preferred grabber.\n\n";
-    set_preferred(choose_grabber());
+    print "\nRandomly selecting grabber order.\n\n";
+    set_order();
 
     show_channels();
@@ -657 +757 @@
 sub status
 {
-    print "    Grabber               Version  Enabled  Ready    Last Data\n" .
-          "    -----------------------------------------------------------\n";
+    print " Grabber                     Version  Enabled  Ready    Last Data\n" .
+          " -----------------------------------------------------------------\n";
     my $star;
-    foreach (sort keys %$grabbers)
+    foreach (sort { $grabbers->{$a}->{order} <=> $grabbers->{$b}->{order} } keys %$grabbers)
     {
         my $h = $grabbers->{$_};
-        printf  "%-3s %-20s %8s %5s %7s %10s\n",
-                $_ eq $preferred ? '(*)' : '',
-                $_,
+        printf  " %-26s %8s %5s %7s %10s\n",
+                "$h->{order}. $_",
                 $h->{ver},
                 $h->{disabled} ? '' : 'Y',
@@ -672 +771 @@
     }
 
-    print "\n(*) = Preferred\n";
+    printf "Grabbers shown in order of preference.\n";
 }
 
@@ -687 +786 @@
     --show-config         Print setup details
 
-    --prefer <s>          Designate <s> as your preferred grabber
+    --setorder <s>        Set order of grabbers to <s> (comma-seperated list of grabbers)
     --disable <s>         Don't ever use grabber <s>
     --enable <s>          Okay, maybe use it again then

status

@@ -1 @@
-shepherd:0.2.3
+shepherd:0.2.5ng
 rex:3.0.1
 ltd:1.51