Changeset 113

Timestamp:

10/21/06 17:49:36 (7 years ago)

Author:

max

Message:

Prevent broken C1 grabbers from causing infinite loops.

Files:

• shepherd (modified) (16 diffs)

shepherd

@@ -118 +118 @@
 my $days = 7;
 my $timeslice;
+my $grabbed;
 my $gmt_offset;
 my $grabber_found_all_data;
@@ -276 +277 @@
         {
             print "CAT1 grabber: grabbing timeslice.\n" if ($debug);
+
+            record_requested_chandays($grabber, $timeslice);
+
             if ($timeslice->{start} != 1)
             {
@@ -342 +346 @@
 
             my $missing_after = convert_chandays_to_list(detect_missing_data());
-            # print "Missing before: " . Dumper($missing_before) . "\nMissing after:" . Dumper($missing_after);
+#           print "Missing before: " . Dumper($missing_before) . "\nMissing after:" . Dumper($missing_after);
             my $list = List::Compare->new($missing_before, $missing_after);
             my @grabbed = $list->get_symmetric_difference();
@@ -393 +397 @@
     # score grabbers
     my $total = score_grabbers();
-    return undef unless ($total);
     
     if ($debug)
     {
         print "Grabber selection probabilities:\n";
+        my $perc;
         foreach (sort { $gscore->{$b} <=> $gscore->{$a} } keys %$gscore)
         {
+            $perc = 100 * ($total ? $gscore->{$_} / $total : 0);
             printf "%25s %6.1f%% %12s\n", 
-                   $_, 100 * $gscore->{$_} / $total, "($gscore->{$_} pts)";
-        }
-    }
-
-    return undef unless (scalar keys %$gscore);
+                   $_, $perc, "($gscore->{$_} pts)";
+        }
+    }
+
+    return undef unless ($total);
 
     my $r = int(rand($total));
@@ -415 +420 @@
         if ($r >= $c and $r < ($c + $gscore->{$grabber}))
         {
-            delete $gscore->{$grabber};
             print "Selected $grabber.\n" if ($debug);
             if ($grabber =~ /(.*) \[cache\]/)
@@ -454 +458 @@
     my $missing = detect_missing_data();
 
-    calculate_best_timeslice($missing);
+    $timeslice = calculate_best_timeslice($missing);
     
     print "Best timeslice: " . Dumper($timeslice) . "\n" if ($debug);
@@ -476 +480 @@
         if ($cat == 1)
         {
-            $key = $missing_slice;
+            $key = cut_down($grabber, $missing_slice);
             print "Grabber $grabber is Category 1: comparing capability to best timeslice.\n" if ($debug);
         }
@@ -591 +595 @@
     my $hits = 0;
 
+#    print "Find cache hits in: " . Dumper($missing);
     foreach my $day (keys %$missing)
     {
@@ -596 +601 @@
         foreach my $ch (@{$missing->{$day}})
         {
-            $hits++ if (grep(/^$date:$ch$/, @{$components->{$grabber}->{cached}}));
+            if (grep(/^$date:$ch$/, @{$components->{$grabber}->{cached}}))
+            {
+                $hits++;
+                print "Hit: $day:$ch\n" if ($debug);
+            }
         }
     }
@@ -616 +625 @@
     my $timeslots_per_day = (24 * 60 * 60) / $policy{timeslot_size};
 
-    # print "Channel data:\n" . Dumper($channel_data);
     foreach my $ch (keys %$channels)
     {
@@ -656 +664 @@
 }
 
+# Takes a hash of what's missing in the format:
+#   { '1' => [ 'ABC', 'SBS' ], '2' => [ 'ABC' ], ... }
+# and sets $timeslice to the biggest 'slice' of that.
+# For example, the biggest slice of the above is days 1 -2 of ABC.
 sub calculate_best_timeslice
 {
@@ -661 +673 @@
 
     my ($overlap, $a);
-    $timeslice = { 'chandays' => 0 };
+    my $slice = { 'chandays' => 0 };
     foreach my $day (1 .. $days)
     {
-        consider_slice($day, $day, @{$missing->{$day}});
+        consider_slice($slice, $day, $day, @{$missing->{$day}});
         $overlap = $missing->{$day};
         foreach my $nextday (($day + 1) .. $days)
@@ -670 +682 @@
             last unless ($missing->{$nextday});
             $a = Algorithm::Diff::LCS($overlap, $missing->{$nextday});
-            # print "Overlap: " . Dumper($a);
             last unless ($a and @{$a});
-            consider_slice($day, $nextday, @{$a});
+            consider_slice($slice, $day, $nextday, @{$a});
             $overlap = $a;
         }
     }
+    return $slice;
 }
 
 sub consider_slice
 {
-    my ($startday, $stopday, @chans) = @_;
+    my ($slice, $startday, $stopday, @chans) = @_;
 
     my $challenger = ($stopday - $startday + 1) * scalar(@chans);
-    return unless ($challenger > $timeslice->{chandays});
+    return unless ($challenger > $slice->{chandays});
 
     # We have a winner!
-    $timeslice->{start} = $startday;
-    $timeslice->{stop} = $stopday;
-    $timeslice->{chans} = [ @chans ];
-    $timeslice->{chandays} = $challenger;
-}
-
+    $slice->{start} = $startday;
+    $slice->{stop} = $stopday;
+    $slice->{chans} = [ @chans ];
+    $slice->{chandays} = $challenger;
+}
+
+# Turns the current $timeslice into a missing-style hash
+# (i.e. { '1' => [ 'ABC', 'SBS' ], '2' => ...
 sub create_missing_slice
 {
@@ -703 +717 @@
 }
 
+# Record what a cacheable C1 grabber has just retrieved for us,
+# so we know next time that this data can be grabbed quickly.
 sub record_cached
 {
@@ -732 +748 @@
 }
 
-# Takes a 'missing' hash and returns it as a list like this:
+# Takes a missing-style hash and returns it as a list like this:
 # ( "20061018:ABC", "20061018:Seven", ... )
 sub convert_chandays_to_list
@@ -751 +767 @@
 }
 
+# If we're about to re-try a grabber, make sure that we're not asking
+# it for the same data. That is, prevent a broken C1 grabber causing
+# an infinite loop.
+sub record_requested_chandays
+{
+    my ($grabber, $slice) = @_;
+
+    print "Recording timeslice request; will not request these chandays from $grabber again.\n" if ($debug);
+
+    my @requested;
+    for my $day ($slice->{start} .. $slice->{stop})
+    {
+        foreach my $ch (@{$slice->{chans}})
+        {
+            push @requested, "$day:$ch";
+        }
+    }
+    if ($grabbed->{$grabber})
+    {
+        push @{$grabbed->{$grabber}}, @requested;
+    }
+    else
+    {
+        $grabbed->{$grabber} = [ @requested ];
+    }
+}
+
+# If this grabber has been called previously, remove those chandays
+# from the current request -- we don't want to ask it over and over
+# for a timeslice that it has already failed to provide.
+sub cut_down
+{
+    my ($grabber, $slice) = @_;
+
+    return $slice unless ($grabbed->{$grabber});
+
+    if ($debug)
+    {
+        print "Grabber $grabber has been called previously; removing " .
+              "requested chandays from timeslice.\n";
+    }
+
+    my $newslice = {};
+    foreach my $day (keys %$slice)
+    {
+        my @chans;
+        foreach my $ch (@{$slice->{$day}})
+        {
+            unless (grep(/$day:$ch/, @{$grabbed->{$grabber}}))
+            {
+                push (@chans, $ch);
+            }
+        }
+        if (@chans)
+        {
+            $newslice->{$day} = [ @chans ];
+        }
+    }
+    return $newslice;
+}
 
 # interpret xmltv data from this grabber/postprocessor