Changeset 30 for shepherd

Timestamp:

10/06/06 00:42:49 (7 years ago)

Author:

lincoln

Message:

first pass at reconciler integration

Files:

• shepherd (modified) (14 diffs)

shepherd

@@ -3 +3 @@
 # "Shepherd"
 
-my $version = '0.2.7';
+my $version = '0.2.8';
 
 # A wrapper for various Aussie TV guide data grabbers
@@ -22 +22 @@
 # 0.2.6   : Postprocessor support
 # 0.2.7   : Changed online file structure
+# 0.2.8   : Integrated reconciler
 
 BEGIN { *CORE::GLOBAL::die = \&my_die; }
@@ -58 +59 @@
 my $ARCHIVE_DIR = "$CWD/archive";
 
+
+#### analyzer settings ####
+
 my $timeslot_size = (15 * 60);                  # 15 minute slots
 my $channel_ok_threshold_percent = 90;          # 90% these may need to be 
@@ -67 +71 @@
                                                 # in a row before it is 
                                                 # automatically disabled
+
+#### reconciler settings ####
+my %reclogic;
+
+# when setting the upper-bounds of a timeslot to look for overlapping
+# programmes between grabbers, cap the upper time window that we are prepared
+# to tolerate programmes ending within.
+# disabled by default (max overrides percent for upper bounds)
+
+$reclogic{compare_overlapping_programmes_extra_overtime_duration_percent} = 5;
+$reclogic{compare_overlapping_programmes_extra_overtime_max} = 0;
+
+# if there is a conflict between two grabbers, choose the grabber
+# with the MOST programmes in the timeslot WITHOUT COUNTING
+# any programmes of <= $reclogic{min_time_override_for_duplicate}
+
+$reclogic{min_time_override_for_duplicate} = (5 * 60);  # default = 5 minutes
+
+# when we've selected one grabber to insert data from, we look at all the other
+# grabbers for overlapping fields we can use to augment our primary grabber
+# set some thresholds for when to consider programming to be the 'same'
+# (with fuzz -/+ 5 mins max), -/+ 2.5mins for programmes <= 15 mins
+
+$reclogic{duplicate_programme_augment_data_short_cutoff} = (15*60); # <=15mins = short, >15mins = long
+$reclogic{duplicate_programme_augment_data_llong_duration_threshold} = (5*60); # +/-5 mins
+$reclogic{duplicate_programme_augment_data_short_duration_threshold} = (2.5*60); # +/- 2min30
+
+# for any programmes we didn't choose, reset the starttime up to a maximum
+# window of 5 minutes to the same value as our "chosen" endtime.
+# if can't readjust, then ignore (remove) that programme
+
+$reclogic{readjust_starttime_for_nonmatched_programmes} = (5*60);
+
+#### end reconciler settings ####
 
 my $opt;
@@ -89 +127 @@
 my $channel_data = { };
 my $starttime, my $endtime;
-my $input_postprocess_files = "";
+my $input_postprocess_file = "$CWD/output_pre_postprocessors.xml";
 my $insufficient_grabber_data = 0;
 
@@ -182 +220 @@
     calc_date_range();
     grab_data();
+    reconcile_data();
     postprocess_data();
     output_data();
@@ -218 +257 @@
 
         my $output = "$GRABBER_DIR/$grabber/output.xmltv";
-        $input_postprocess_files .= "$output ";
 
         my $comm = "$GRABBER_DIR/$grabber/$grabber " .
@@ -353 +391 @@
 
                 next if (!$t1 || !$t2); # if we can't parse stop/start then clearly THIS data is bunk!
+
+                # store t1 and t2 times in the xmltv data for later on (shh.. ton't tell anyone..)
+                $prog->{start_epoch} = $t1->epoch;
+                $prog->{stop_epoch} = $t2->epoch;
 
                 # store plugin-specific stats
@@ -482 +524 @@
 
 # -----------------------------------------
+# Subs: Reconciling data
+# -----------------------------------------
+
+# for all the data we have, try to pick the best bits!
+sub reconcile_data
+{
+    printf "\nReconciling data:\n\n";
+
+    my @proglist = [ ];
+    my @position_pointer = [ ];
+    my $num_grabbers = 0;
+    my $pref_order;
+
+    printf "Preference for whose data we prefer as follows:\n";
+    foreach my $proggy (sort { $grabbers->{$a}->{order} <=> $grabbers->{$b}->{order} } keys %$grabbers) {
+        if ((!$grabbers->{$proggy}->{disabled}) && ($plugin_data->{$proggy}) && ($plugin_data->{$proggy}->{valid})) {
+            my $orig_prog = $plugin_data->{$proggy}->{xmltv}->[3];
+            my $prognum = 0;
+            foreach my $new_prog (sort order_channel_time @{$orig_prog}) {
+                $proglist[$num_grabbers]->[$prognum] = $new_prog;
+                $prognum++;
+            }
+
+            printf "  %d. %s (%d programmes)\n",($num_grabbers+1),$proggy,$prognum;
+            $num_grabbers++;
+        }
+    }
+
+    my %writer_args = ( encoding => 'ISO-8859-1' );
+    my $fh = new IO::File(">input_postprocess_file") || die "can't open outputfile $input_postprocess_file: $!";
+    $writer_args{OUTPUT} = $fh;
+    my $writer = new XMLTV::Writer(%writer_args);
+
+    $writer->start({'source-info-url' => "about:blank", 'source-info-name' => "$progname $version", 'generator-info-name' => "$progname $version"} );
+
+    for my $ch (sort keys %$channels) {
+        $writer->write_channel({'display-name' => [[ $ch, $langs ]], 'id' => $channels->{$ch}} );
+    }
+
+    for my $ch (sort keys %$channels) {
+        printf "Reconciling channel: %s (%s)\n",$ch,$channels->{$ch};
+
+        #
+        # 1. position pointers to first piece of data for this channel
+        #
+
+        printf "REC#1: processing channel %s\n",$channels->{$ch} if $debug;
+        my $position_pointer;
+        for (my $i=0; $i < $num_grabbers; $i++) {
+            $position_pointer[$i] = 0;
+            while ((defined $proglist[$i]->[($position_pointer[$i])]) && ($proglist[$i]->[($position_pointer[$i])]->{'channel'} ne $channels->{$ch})) {
+                $position_pointer[$i]++;
+            }
+            if (!defined $proglist[$i]->[($position_pointer[$i])]) {
+                $position_pointer[$i] = -1;
+                printf "REC#1: no programmes found for channel %s from gradder %d\n",$channels->{$ch},$i if $debug;
+            } else {
+                printf "REC#1: advanced position_pointer to %d for grabber %d (first programme is \"%s\", start %d, stop %d)\n",
+                    $position_pointer[$i],$i,${XMLTV::best_name($langs,$proglist[$i]->[($position_pointer[$i])]->{title})}[0],
+                    $proglist[$i]->[($position_pointer[$i])]->{start_epoch},$proglist[$i]->[($position_pointer[$i])]->{stop_epoch} if $debug;
+            }
+        }
+
+        my $all_done_on_this_channel;
+        do {
+            #
+            # 2. find 'earliest' programme from all the choices
+            #
+
+            $all_done_on_this_channel = 1; # unless proven otherwise
+            my $earliest_programme_time = undef;
+            my $earliest_programme_slot = undef;
+            for (my $i=0; $i < $num_grabbers; $i++) {
+                next if ($position_pointer[$i] == -1); # skip if no programmes on this channel from this grabber
+                my $this_programme = $proglist[$i]->[($position_pointer[$i])];
+
+                if ((!defined $this_programme) || (!defined $this_programme->{title}) || ($this_programme->{channel} ne $channels->{$ch})) {
+                    # no more programmes on this channel for this grabber!
+                    printf "REC#2: no more programmes on grabber %d for this channel\n",$i if $debug;
+                    $position_pointer[$i] = -1;
+                } else {
+                    if ((!defined $earliest_programme_time) || ($earliest_programme_time > $this_programme->{'start_epoch'})) {
+                        $earliest_programme_time = $this_programme->{'start_epoch'};
+                        $earliest_programme_slot = $i;
+                        printf "REC#2: earliest programme (so far) on grabber %d, start %d (end %d) programme \"%s\"\n",
+                            $i,$this_programme->{start_epoch},$this_programme->{stop_epoch},${XMLTV::best_name($langs,$this_programme->{title})}[0] if $debug;
+                    } else {
+                        printf "REC#2:    programme on grabber %d was not earlier start %d (end %d) programme \"%s\"\n",
+                            $i,$this_programme->{start_epoch},$this_programme->{stop_epoch},${XMLTV::best_name($langs,$this_programme->{title})}[0] if $debug;
+                    }
+                }
+            }
+            if (!defined $earliest_programme_slot) {
+                # no programmes available on ANY grabber for this channel, skip this channel
+                printf "REC#2: no programmes on any grabbers for channel %s, skipping this channel\n",$channels->{$ch} if $debug;
+            } else {
+                #
+                # 3a. compare how many programmes on other grabbers overlap with it
+                # TODO (FUTURE): enhance this to do "majority voting" acrosa all grabbers based on identical start/stop
+                # TODO (FUTURE): where star/stop times match exactly but title names dont, record the mapping for known-as_to_most-preferred-title so we can use this in future transformations
+
+                my $preferred_programme = $proglist[$earliest_programme_slot]->[($position_pointer[$earliest_programme_slot])];
+                my $startpoint = $preferred_programme->{'start_epoch'};
+                my $stoppoint = $preferred_programme->{'stop_epoch'};
+                my $duration = $stoppoint - $startpoint;
+                my $extraduration = 0;
+                $extraduration = int($duration * ($reclogic{compare_overlapping_programmes_extra_overtime_duration_percent}/100)) if ($reclogic{compare_overlapping_programmes_extra_overtime_duration_percent} > 0);
+                $extraduration = $reclogic{compare_overlapping_programmes_extra_overtime_max} if ($extraduration > $reclogic{compare_overlapping_programmes_extra_overtime_max}); # upper limit
+                $stoppoint += $extraduration;
+                printf "REC#3: comparing other grabbers to see how many programmes fit in timeslot %d to %d (%d+%d)\n",$startpoint,$stoppoint,$duration,$extraduration if $debug;
+
+                my $max_progs_found = 0;
+                my $max_progs_slot = undef;
+                for (my $i=0; $i < $num_grabbers; $i++) {
+                    next if ($i == $earliest_programme_slot);
+                    next if ($position_pointer[$i] == -1);
+                    my $position_offset = 0;
+                    my $progs_found = 0;
+
+                    while ((defined $proglist[$i]->[($position_pointer[$i]+$position_offset)]) &&
+                           ($proglist[$i]->[($position_pointer[$i]+$position_offset)]->{'channel'} eq $channels->{$ch}) &&
+                           ($proglist[$i]->[($position_pointer[$i]+$position_offset)]->{'start_epoch'} >= $startpoint) &&
+                           ($proglist[$i]->[($position_pointer[$i]+$position_offset)]->{'stop_epoch'} <= $stoppoint)) {
+                        my $this_prog_duration = $proglist[$i]->[($position_pointer[$i]+$position_offset)]->{'stop_epoch'} - $proglist[$i]->[($position_pointer[$i]+$position_offset)]->{'start_epoch'};
+
+                        printf "REC#3a.  programme on grabber %d matched (start %d end %d), \"%s\"%s\n", $i,
+                            $proglist[$i]->[($position_pointer[$i]+$position_offset)]->{'start_epoch'},
+                            $proglist[$i]->[($position_pointer[$i]+$position_offset)]->{'stop_epoch'},
+                            ${XMLTV::best_name($langs,$proglist[$i]->[($position_pointer[$i]+$position_offset)]->{'title'})}[0],
+                            ($this_prog_duration <= $reclogic{min_time_override_for_duplicate} ? 
+                            ", but ignored because of min_time_override_for_duplicate ($reclogic{min_time_override_for_duplicate} sec)" : "") if $debug;
+
+                        $progs_found++ if ($this_prog_duration > $reclogic{min_time_override_for_duplicate});
+                        $position_offset++;
+                    }
+
+                    printf "REC#3a:  %d programmes on grabber %d within timeslot\n",$progs_found,$i if $debug;
+                    if ($progs_found > $max_progs_found) {
+                        $max_progs_found = $progs_found;
+                        $max_progs_slot = $i;
+                    }
+                }
+
+                # 3b. if there are 2 or more programmes on other channels, use those - otherwise use this one
+                if ((!defined $max_progs_slot) || ($max_progs_found <= 1)) {
+                    printf "REC#3b: no grabbers with more programmes in timeslot found\n" if $debug;
+                } else {
+                    printf "REC#3b: grabber %d has %d programmes between %d and %d - using THAT grabber as our preference now!\n",
+                        $max_progs_slot,$max_progs_found,$startpoint,$stoppoint if $debug;
+                    $earliest_programme_slot = $max_progs_slot;
+                }
+
+                #
+                # 4. populate our "reconciled" programme list with the programming chosen
+                #
+
+                my $chosen_prog = $proglist[$earliest_programme_slot]->[($position_pointer[$earliest_programme_slot])];
+                $startpoint = $chosen_prog->{'start_epoch'};
+                $stoppoint = $chosen_prog->{'stop_epoch'};
+                my $new_prog_entry = $chosen_prog;
+
+                printf "REC#4: chosen programme is from grabber %d: start %d, end %d, duration %d: \"%s\"\n",
+                    $earliest_programme_slot,$startpoint,$stoppoint,($stoppoint-$startpoint),
+                    ${XMLTV::best_name($langs,$chosen_prog->{'title'})}[0] if $debug;
+
+                #
+                # 5a. see if we have it duplicated from multiple grabbers (with fuzz -/+ 5 mins max), -/+ 2.5mins for programmes <= 15 mins
+                # TODO (FUTURE): should really do an exact start/stop match as a first pass, then do the fuzz afterwards..
+
+                my $start1, my $start2, my $stop1, my $stop2;
+
+                if (($stoppoint - $startpoint) <= $reclogic{duplicate_programme_augment_data_short_cutoff}) {
+                    $start1 = $startpoint - $reclogic{duplicate_programme_augment_data_short_duration_threshold};
+                    $start2 = $startpoint + $reclogic{duplicate_programme_augment_data_short_duration_threshold};
+                    $stop1 = $stoppoint - $reclogic{duplicate_programme_augment_data_short_duration_threshold};
+                    $stop2 = $stoppoint + $reclogic{duplicate_programme_augment_data_short_duration_threshold};
+                } else {
+                    $start1 = $startpoint - $reclogic{duplicate_programme_augment_data_long_duration_threshold};
+                    $start2 = $startpoint + $reclogic{duplicate_programme_augment_data_long_duration_threshold};
+                    $stop1 = $stoppoint - $reclogic{duplicate_programme_augment_data_long_duration_threshold};
+                    $stop2 = $stoppoint + $reclogic{duplicate_programme_augment_data_long_duration_threshold};
+                }
+                if ($start2 >= $stop1) {
+                    $start2 = $startpoint;
+                    $stop1 = $stoppoint;
+                }
+
+                printf "REC#5: looking in other grabbers for matching programmes within timeslot start %d-%d (%d) and end %d-%d (%d)\n",
+                    $start1,$start2,($start2-$start1),$stop1,$stop2,($stop2-$stop1) if $debug;
+
+                for (my $i=0; $i < $num_grabbers; $i++) {
+                    next if ($i == $earliest_programme_slot);
+                    next if ($position_pointer[$i] == -1);
+
+                    if ((defined $proglist[$i]->[($position_pointer[$i])]) &&
+                        ($proglist[$i]->[($position_pointer[$i])]->{'channel'} eq $channels->{$ch}) &&
+                        ($proglist[$i]->[($position_pointer[$i])]->{'start_epoch'} >= $start1) &&
+                        ($proglist[$i]->[($position_pointer[$i])]->{'start_epoch'} < $start2) &&
+                        ($proglist[$i]->[($position_pointer[$i])]->{'stop_epoch'} >= $stop1) &&
+                        ($proglist[$i]->[($position_pointer[$i])]->{'stop_epoch'} < $stop2)) {
+                        # winner .. matches our criteria ...
+                        my $match_prog = $proglist[$i]->[($position_pointer[$i])];
+
+                        printf "REC#5:   found programme on grabber %d: start %d, end %d: \"%s\"\n", $i,
+                            $match_prog->{'start_epoch'},$match_prog->{'stop_epoch'},
+                            ${XMLTV::best_name($langs,$match_prog->{'title'})}[0] if $debug;
+
+                        foreach my $field (keys %{$match_prog}) {
+                            # 5b. pick fields from each one in order of our preferences
+                            next if ($field eq "start_epoch");
+                            next if ($field eq "stop_epoch");
+                            if (!defined $new_prog_entry->{$field}) {
+                                printf "REC#5b:      adding field \"%s\"\n",$field;
+                                $new_prog_entry->{$field} = $match_prog->{$field};
+                                # TODO (FUTURE): should we add to programme description to say where we got what data from?
+                            }
+                        }
+                    }
+                }
+
+                # 6.  write out new entry
+                printf "REC#6: writing out programme entry\n" if $debug;
+                #delete $new_prog_entry->{'start_epoch'};
+                #delete $new_prog_entry->{'stop_epoch'};
+                &cleanup($new_prog_entry);
+                $writer->write_programme($new_prog_entry);
+
+                # 7a. remove all programmes that end before this endtime
+                for (my $i=0; $i < $num_grabbers; $i++) {
+                    next if ($position_pointer[$i] == -1);
+                    while ((defined $proglist[$i]->[($position_pointer[$i])]) &&
+                           ($proglist[$i]->[($position_pointer[$i])]->{'channel'} eq $channels->{$ch}) &&
+                           ($proglist[$i]->[($position_pointer[$i])]->{'stop_epoch'} <= $stoppoint)) {
+                        printf "REC#7a: removing programme on grabber %d slot %d since it ends before inserted start: start %d end %s: \"%s\"\n",
+                            $i, $position_pointer[$i],
+                            $proglist[$i]->[($position_pointer[$i])]->{'start_epoch'},
+                            $proglist[$i]->[($position_pointer[$i])]->{'stop_epoch'},
+                            ${XMLTV::best_name($langs,$proglist[$i]->[($position_pointer[$i])]->{'title'})}[0] if $debug;
+                        delete $proglist[$i]->[($position_pointer[$i])];
+                        $position_pointer[$i]++;
+                    }
+                }
+
+                # 7b. adjust starttimes of any programmes to match endtime (with fuzz of +5 mins max)
+                for (my $i=0; $i < $num_grabbers; $i++) {
+                    next if ($position_pointer[$i] == -1);
+                    my $position_offset = 0;
+                    while ((defined $proglist[$i]->[($position_pointer[$i]+$position_offset)]) &&
+                           ($proglist[$i]->[($position_pointer[$i]+$position_offset)]->{'channel'} eq $channels->{$ch}) &&
+                           ($proglist[$i]->[($position_pointer[$i]+$position_offset)]->{'start_epoch'} < $stoppoint)) {
+                        my $this_prog = $proglist[$i]->[($position_pointer[$i]+$position_offset)];
+                        if (($this_prog->{'start_epoch'} + $reclogic{readjust_starttime_for_nonmatched_programmes}) >= $stoppoint) {
+                            printf "REC#7b: adjusting starttime on grabber %d slot %d since it starts <5mins before inserted end:\n",
+                                $i,($position_pointer[$i]+$position_offset) if $debug;
+                            printf "REC#7b:    orig: start %d end %d, now: start %d end %d: \"%s\"\n",
+                                $this_prog->{'start_epoch'}, $this_prog->{'stop_epoch'}, $stoppoint, $this_prog->{'stop_epoch'},
+                                ${XMLTV::best_name($langs,$this_prog->{'title'})}[0] if $debug;
+                            $proglist[$i]->[($position_pointer[$i]+$position_offset)]->{'start_epoch'} = $stoppoint;
+                            $proglist[$i]->[($position_pointer[$i]+$position_offset)]->{'start'} = sprintf "%s",(strftime "%Y%m%d%H%M", localtime($stoppoint));
+                            $position_offset++;
+                        } else {
+                            printf "REC#7b: removing grabber %d slot %d programme because it started too long before chosen programme: start %d end %d: \"%s\"\n",
+                                $i, ($position_pointer[$i]+$position_offset), $this_prog->{'start_epoch'}, $this_prog->{'stop_epoch'},
+                                ${XMLTV::best_name($langs,$this_prog->{'title'})}[0] if $debug;
+                            $position_pointer[$i]++;
+                        }
+                    }
+                }
+            }
+
+            # 8. check that we still have at least one pointer on current channel
+            for (my $i=0; $i < $num_grabbers; $i++) {
+                next if ($position_pointer[$i] == -1);
+                printf "REC#8: grabber %d is now at slot %d\n",$i,$position_pointer[$i] if $debug;
+                if ((defined $proglist[$i]->[($position_pointer[$i])]) &&
+                    ($proglist[$i]->[($position_pointer[$i])]->{'channel'} eq $channels->{$ch})) {
+                    $all_done_on_this_channel = 0;
+                    printf "REC#8:   grabber %d is at slot %d is still on channel \"%s\"\n",$i,$position_pointer[$i],$channels->{$ch} if $debug;
+                }
+            }
+            printf "REC#9:\n" if $debug;
+        } until ($all_done_on_this_channel);
+    }
+    $writer->end();
+}
+
+# sorting helper routine - sort by channel then by start-time
+sub order_channel_time {
+        my $chanresult = $a->{channel} cmp $b->{channel};
+        return $chanresult if ($chanresult != 0);
+        return ($a->{start_epoch} <=> $b->{start_epoch});
+}
+
+# descend a structure and clean up various things, including stripping
+# leading/trailing spaces in strings, translations of html stuff etc
+#   -- taken & modified from Michael 'Immir' Smith's excellent tv_grab_au
+my %amp;
+BEGIN { %amp = ( nbsp => ' ', qw{ amp & lt < gt > apos ' quot " } ) }
+sub cleanup {
+    my $x = shift;
+    if    (ref $x eq "REF")   { cleanup($_) }
+    elsif (ref $x eq "HASH")  { cleanup(\$_) for values %$x }
+    elsif (ref $x eq "ARRAY") { cleanup(\$_) for @$x }
+    elsif (defined $$x) {
+        $$x =~ s/&(#(\d+)|(.*?));/ $2 ? chr($2) : $amp{$3}||' ' /eg;
+        # $$x =~ s/[^\x20-\x7f]/ /g;
+        $$x =~ s/(^\s+|\s+$)//g;
+    }
+}
+
+
+# -----------------------------------------
 # Subs: Postprocessing
 # -----------------------------------------
@@ -492 +846 @@
     # reverts back to the previous postprocessor if it was shown to be bad
 
-    # first time around: feed in $input_postprocess_files
+    # first time around: feed in reconciled data ($input_postprocess_file)
+
     my $need_more_data;
 
@@ -514 +869 @@
         $comm .= " --debug" if ($debug);
         $comm .= " @ARGV" if (@ARGV);
-        $comm .= " $input_postprocess_files";
+        $comm .= " $input_postprocess_file";
         print "SHEPHERD: Excuting command: $comm\n";
 
@@ -562 +917 @@
             # accept what this postprocessor did to our output ...
             printf "SHEPHERD: accepting output from postprocessor %s, feeding it into next stage\n",$postprocessor;
-            $input_postprocess_files = $output;
+            $input_postprocess_file = $output;
             delete $postprocessors->{$postprocessor}->{conescutive_failures} if (defined $postprocessors->{$postprocessor}->{conescutive_failures});
         }
@@ -571 +926 @@
 sub output_data
 {
-    # $input_postprocess_files (hopefully just one file now) contains our final output
+    # $input_postprocess_file contains our final output
     # send it to whereever --output told us to!
 
@@ -577 +932 @@
     $output_filename = $opt->{output} if ($opt->{output});
 
-    open(F,">$output_filename") || die "could not open output file $output_filename for writing: $!\n";
-
-    foreach my $infile (split(/ /,$input_postprocess_files)) {
-        if (!(open(INFILE,"<$infile"))) {
-            printf "WARNING: could not open input file \"%s\": %s\n", $infile, $!;
-            printf "Output XMLTV data may be damanged as a result!\n";
-        } else {
-            while (<INFILE>) {
-                print F $_ if ($opt->{output});
-            }
-            close(INFILE);
-        }
-    }
-    close(F);
+    open(OUTFILE,">$output_filename") || die "could not open output file $output_filename for writing: $!\n";
+
+    if (!(open(INFILE,"<$input_postprocess_file"))) {
+        printf "WARNING: could not open input file \"%s\": %s\n", $input_postprocess_file, $!;
+        printf "Output XMLTV data may be damanged as a result!\n";
+    } else {
+        while (<INFILE>) {
+            print OUTFILE $_;
+        }
+        close(INFILE);
+        close(OUTFILE);
+    }
 
     printf "Final output stored in $output_filename.\n";