Changeset 96

Timestamp:

10/18/06 18:32:22 (7 years ago)

Author:

max

Message:

More intelli-random grabber ordering: keeps track of what C1 grabbers have cached and favors that

Files:

• shepherd (modified) (22 diffs)

shepherd

@@ -43 +43 @@
 use DateTime::Format::Strptime;
 use POSIX qw(strftime);
+use Date::Manip;
 use Time::HiRes qw(gettimeofday tv_interval);
 use Algorithm::Diff;
+use List::Compare;
 
 # ---------------------------------------------------------------------------
@@ -116 +118 @@
 
 print ucfirst($progname) . " v$version\n\n";
-#print "Cwd: $CWD.\n";
 
 # Any options Shepherd doesn't understand, we'll pass to the grabber(s)
@@ -224 +225 @@
     my $found_data_percent = 0;
 
-    print "\nGrabber stage:\n";
+    print "\nGrabber stage.\n";
 
     my $grabber;
@@ -230 +231 @@
     while ($grabber = choose_grabber())
     {
+
+        my $missing_before;
+
         $used_grabbers++;
 
@@ -247 +251 @@
         # very little cost in grabbing that extra data, and we can use it in the reconciler
         # to verify that everything looks OK.
-        if ($components->{$grabber}->{config}->{category} == 1)
-        {
-            print "CAT1 grabber: grabbing timeslice.\n";
+        if (query_config($grabber, 'category') == 1)
+        {
+            print "CAT1 grabber: grabbing timeslice.\n" if ($debug);
             if ($timeslice->{start} != 1)
             {
                 $comm .= " " . 
-                         $components->{$grabber}->{config}->{option_offset} .
+                         query_config($grabber, 'option_offset') .
                          " " .
                          ($timeslice->{start} - 1);
@@ -261 +265 @@
             if ($timeslice->{start} != 1 
                     and 
-                !$components->{$grabber}->{config}->{option_offset_eats_days})
+                !query_config($grabber, 'option_offset_eats_days'))
             {
                 $n -= $timeslice->{start};
             }
             $comm .= " " .
-                     $components->{$grabber}->{config}->{option_days} .
+                     query_config($grabber, 'option_days') .
                      " " . 
                      $n;
@@ -279 +283 @@
             write_file($tmpcf, 'temporary channels', [ $tmpchans ], [ 'channels' ]);
             $comm .= " --channels_file $tmpcf";
+
+            if (query_config($grabber, 'cache'))
+            {
+                $missing_before = convert_chandays_to_list(detect_missing_data());
+            }
         }
         else
         {
+            print "CAT2 grabber: grabbing everything.\n" if ($debug);
             $comm .= " --days $days" if ($days);
             $comm .= " --offset $opt->{offset}" if ($opt->{offset});
@@ -306 +316 @@
 
         last if ($found_data_percent >= $channel_ok_threshold_percent);
+
+        # Record what we grabbed for cacheable C1 grabbers
+        if ($missing_before)
+        {
+
+            my $missing_after = convert_chandays_to_list(detect_missing_data());
+            print "Missing before: " . Dumper($missing_before) . "\nMissing after:" . Dumper($missing_after);
+            my $list = List::Compare->new($missing_before, $missing_after);
+            my @grabbed = $list->get_symmetric_difference();
+            print "Difference: " . Dumper(\@grabbed);
+            record_cached($grabber, @grabbed);
+            write_config_file();
+        }
     }
 
@@ -337 +360 @@
             {
                 $gscore->{$_} = 0;
-# Cache stuff: not enabled yet
-#               if ($components->{$_}->{config}->{category} == 1
-#                       and
-#                   $components->{$_}->{config}->{cache})
-#               {
-#                   $gscore->{$_ . ' [cache]'} = 0;
-#               }
+                if (query_config($_, 'category') == 1
+                        and
+                    query_config($_, 'cache'))
+                {
+                    $gscore->{$_ . ' [cache]'} = 0;
+                }
             }
         }
@@ -380 +402 @@
             else
             {
-                if ($components->{$grabber}->{config}->{category} == 2)
+                if (query_config($grabber, 'category') == 2)
                 {
                     # We might want to run C1 grabbers multiple times
@@ -417 +439 @@
     my $missing_slice = create_missing_slice();
 
+    my $bestdq = 0;
+
     # So! Compare C2 grabbers against the raw missing file, because we'll get
     # everything. But compare C1 grabbers against the timeslice, because we'll
@@ -426 +450 @@
         
         $hits = 0;
-
-        if ($grabber =~ /(.*) \[cache\]$/)
-        {
-            $hits = find_cache_hits($1, $missing);
-            $cat = 2;
-            $dq = $components->{$1}->{config}->{quality};
+        $cat = query_config($grabber, 'category');
+        $dq = query_config($grabber, 'quality');
+
+        if ($grabber =~ /\[cache\]/)
+        {
+            $hits = find_cache_hits($grabber, $missing);
         }
         else
         {
-            my $key = $missing;
-            if ($components->{$grabber}->{config}->{category} == 1)
+            my $key;
+            if ($cat == 1)
             {
                 $key = $missing_slice;
+                print "Grabber $grabber is Category 1: comparing capability to best timeslice.\n" if ($debug);
+            }
+            else
+            {
+                $key = $missing;
+                print "Grabber $grabber is Category 2: comparing capability to all wanted channels and days.\n" if ($debug);
             }
             foreach my $day (sort keys %$key)
@@ -444 +474 @@
                 my $val = supports_day($grabber, $day);
                 next unless ($val);
-                print "Day $day:";
+                print "Day $day:" if ($debug);
                 foreach my $ch (@{$key->{$day}})
                 {
@@ -455 +485 @@
                 print "\n";
                 $hits = 1 if ($hits > 0 and $hits < 1);
-
-                $cat = $components->{$grabber}->{config}->{category};
-                unless ($cat)
-                {
-                    print "WARNING: Grabber $grabber has no category support ".
-                    "in config.\n";
-                    $cat = 1;
-                }
-
-                $dq = $components->{$grabber}->{config}->{quality};
-                unless ($dq)
-                {
-                    print "WARNING: Grabber $grabber has no quality support ".
-                    "in config.\n";
-                    $dq = 1;
-                }
-            }
-        }
+            }
+        }
+
         $mult = 1;
-        $mult++ if ($cat == 2);
+        $mult += 2 if ($cat == 2);
         $mult *= 2 ** ($dq-1);
+        $score = int($hits * $mult);
 
         $score = int($hits * $mult);
@@ -481 +497 @@
         $gscore->{$grabber} = $score;
         $total += $score;
-    }
+
+        # Keep track of the best quality of valid grabbers
+        if ($score and $dq > $bestdq)
+        {
+            $bestdq = $dq;
+        }
+    }
+
+    # Eliminate grabbers of data quality 1 if there are any quality 2s
+    # or 3s present.
+    foreach (keys %$gscore)
+    {
+        if ($gscore->{$_}
+                and
+            query_config($_, 'quality') == 1
+                and
+            $bestdq > 1)
+        {
+            $total -= $gscore->{$_};
+            $gscore->{$_} = 0;
+            print "Zeroing grabber $_ due to low data quality.\n" if ($debug);
+        }
+    }
+
     return $total;
 }
@@ -489 +528 @@
     my ($grabber, $ch) = @_;
 
-    my $channels_supported = $components->{$grabber}->{config}->{channels};
+    my $channels_supported = query_config($grabber, 'channels');
     unless (defined $channels_supported)
     {
@@ -508 +547 @@
     my ($grabber, $day) = @_;
 
-    return 0 unless ($day <= $components->{$grabber}->{config}->{max_days});
-    return 0.5 if ($day > $components->{$grabber}->{config}->{max_reliable_days});
+    return 0 unless ($day <= query_config($grabber, 'max_days'));
+    return 0.5 if ($day > query_config($grabber, 'max_reliable_days'));
     return 1;
 }
@@ -517 +556 @@
     my ($grabber, $missing) = @_;
 
-    return 5;
+    if ($grabber =~ /(.*) \[cache\]/)
+    {
+        $grabber = $1;
+    }
+
+    return 0 unless ($components->{$grabber}->{cached});
+
+    my $hits = 0;
+
+    foreach my $day (keys %$missing)
+    {
+        my $date = substr(DateCalc("today", "+ " . ($day - 1) . " days"), 0, 8);
+        foreach my $ch (@{$missing->{$day}})
+        {
+            $hits++ if (grep(/^$date:$ch$/, @{$components->{$grabber}->{cached}}));
+        }
+    }
+    return $hits;
 }
 
@@ -525 +581 @@
 # made more fine-grained if we think grabbers will support that.
 #
+# TODO: Currently buggy in that it thinks we need 24 hours worth of
+# data for day 1 when we don't.
+#
 sub detect_missing_data
 {
@@ -531 +590 @@
     my $timeslots_per_day = (24 * 60 * 60) / $timeslot_size;
 
+    # print "Channel data:\n" . Dumper($channel_data);
     foreach my $ch (keys %$channels)
     {
@@ -615 +675 @@
     return $ret;
 }
+
+sub record_cached
+{
+    my ($grabber, @grabbed) = shift;
+
+    print "Recording cache for grabber $grabber.\n" if ($debug);
+
+    my $gcache = $components->{$grabber}->{cached};
+    $gcache = [ ] unless ($gcache);
+    my @newcache;
+    my $today = strftime("%Y%m%d", localtime);
+
+    # remove old chandays
+    foreach my $chanday (@$gcache)
+    {
+        $chanday =~ /(\d+):(.*)/;
+        if ($1 >= $today)
+        {
+            push (@newcache, $chanday);
+        }
+    }
+
+    # record new chandays
+    foreach (@grabbed)
+    {
+        push (@newcache, $_) unless (grep(/^$_$/, @newcache));
+    }
+    $components->{$grabber}->{cached} = [ @newcache ];
+}
+
+# Takes a 'missing' hash and returns it as a list like this:
+# ( "20061018:ABC", "20061018:Seven", ... )
+sub convert_chandays_to_list
+{
+    my $h = shift;
+
+    my @ret;
+    foreach my $day (keys %$h)
+    {
+        my $date = substr(DateCalc("today", "+ " . ($day - 1) . " days"), 0, 8);
+        foreach my $ch (@{$h->{$day}})
+        {
+            push (@ret, "$date:$ch");
+        }
+    }
+    @ret = sort @ret;
+    return \@ret;
+}
+
 
 # interpret xmltv data from this grabber/postprocessor
@@ -1320 +1429 @@
     }
     return @ret;
+}
+
+sub query_config
+{
+    my ($grabber, $key) = @_;
+
+    if ($grabber =~ /(.*) \[cache\]/)
+    {
+        $grabber = $1;
+    }
+    return undef unless ($components->{$grabber});
+    return $components->{$grabber}->{config}->{$key};
 }