Changeset 294

Timestamp:

11/25/06 17:53:57 (6 years ago)

Author:

lincoln

Message:

multiple enhancements to the reconciler:

1. choose_title() now keeps its own history of titles seen and uses that in finding preferred titles. far more robust. addresses #5

2. because of 1., increase $reclogic{title_xlate_table_min_alt_progs} to 1: only store a preferred title if we have at least 1 alternative

3. reconciler can now deal with XMLTV data both with and without timezones in start/stop times

4. reconciler now correctly parses all known 'generator-info-name' fields from input XMLTV files - primarily means its less verbose in logging and what it stores in its title-match history

due to changes 1 and 2, it is recommended that you manually reset your
title history with '~/.shepherd/tv_grab_au --reset'

Files:

• reconcilers/reconciler_mk2 (modified) (13 diffs)
• status (modified) (1 diff)

reconcilers/reconciler_mk2

@@ -21 +21 @@
 #    0.08  20oct06      improved preference-title rewriter, more debugging to logfile
 #    0.09  24oct06      --preftitle support
+#    0.15  26nov06      reworked the title translations a bit
+#                        - now only stores preferred titles when it has seen more than once
+#                        - keeps last 30 days titles (indexed by time-of-day/day-of-week)
+#                          as back history for alternate titles when switching grabbers
 
 #
@@ -91 +95 @@
 
 my $progname = "reconciler_mk2";
-my $version = "0.14";
+my $version = "0.15";
 
 use LWP::UserAgent;
@@ -140 +144 @@
 $reclogic{max_programme_length} = (8 * 60 * 60); # 8 hours
 
-# do store a preference title, even if we don't have any alternatives
-$reclogic{title_xlate_table_min_alt_progs} = 0;
+# store a preferred title only if we have at least 1 alternative
+$reclogic{title_xlate_table_min_alt_progs} = 1;
+
+# keep up to 30 days of title history
+$reclogic{title_history} = 30;
+
+# matching of history titles requires programme start within 20 minutes and duration within 10%
+$reclogic{title_history_start_fuzzy_match_window} = (20*60);    # seconds
+$reclogic{title_history_duration_fuzzy_match_percent} = 10;     # percent
+
+
 
 #
@@ -189 +202 @@
 my $out = { };
 my $w;
+my $gmt_offset;
 
 my $setting_override;
 my %cli_override;
 my $title_xlate_table;  # cached
+my $title_history;      # cached
 
 my %amp;
@@ -363 +378 @@
 sub write_config_file
 {
+        # age out old titles (default is to keep for 30 days as per $reclogic{title_history})
+        for my $cache_key (keys %{$title_history}) {
+                my ($starttime, $duration, $channel, $grabber) = split(/,/,$cache_key);
+                if ($starttime < (time-(86400*$reclogic{title_history}))) {
+                        delete $title_history->{$cache_key};
+                        $stats{expired_old_titles}++;
+                }
+        }
+
         open(CONF, ">$opt->{config_file}") || die "cannot write to $opt->{config_file}: $!";
         print CONF Data::Dumper->Dump(
-                [$setting_override,  $title_xlate_table  ],
-                ["setting_override", "title_xlate_table" ]);
+                [$setting_override,  $title_xlate_table,  $title_history],
+                ["setting_override", "title_xlate_table", "title_history" ]);
         close CONF;
         &log(1,(sprintf "updated configuration file %s.\n",$opt->{config_file}));
@@ -541 +565 @@
 ######################################################################################################
 
+# strptime type date parsing - BUT - if no timezone is present, treat time as being in localtime
+# rather than the various other perl implementation which treat it as being in UTC/GMT
 sub parse_xmltv_date
 {
         my $datestring = shift;
-        my @timeattr; # = localtime(time); # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
-
-        if ($datestring =~ /^(....)(..)(..)(..)(..)/) {
-                ($timeattr[5],$timeattr[4],$timeattr[3],$timeattr[2],$timeattr[1],$timeattr[0]) = (int($1)-1900,int($2)-1,int($3),int($4),int($5),0);
-                ($timeattr[6],$timeattr[7],$timeattr[8]) = (-1,-1,-1);
-                # NOTE: FIXME: we are ignoring timezone for now...
-
-                my $return_epoch = mktime(@timeattr);
-                return $return_epoch if ($return_epoch > 1);
-        }
-
+        my @t; # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
+        my $tz_offset = 0;
+
+        # work out GMT offset - we only do this once
+        if (!$gmt_offset) {
+                my $tzstring = strftime("%z", localtime(time));
+
+                $gmt_offset = (60*60) * int(substr($tzstring,1,2));     # hr
+                $gmt_offset += (60 * int(substr($tzstring,3,2)));       # min
+                $gmt_offset *= -1 if (substr($tzstring,0,1) eq "-");    # +/-
+        }
+
+        if ($datestring =~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})/) {
+                ($t[5],$t[4],$t[3],$t[2],$t[1],$t[0]) = (int($1)-1900,int($2)-1,int($3),int($4),int($5),0);
+                ($t[6],$t[7],$t[8]) = (-1,-1,-1);
+
+                # if input data has a timezone offset, then offset by that
+                if ($datestring =~ /\+(\d{2})(\d{2})/) {
+                        $tz_offset = $gmt_offset - (($1*(60*60)) + ($2*60));
+                } elsif ($datestring =~ /\-(\d{2})(\d{2})/) {
+                        $tz_offset = $gmt_offset + (($1*(60*60)) + ($2*60));
+                }
+
+                my $e = mktime(@t);
+                return ($e+$tz_offset) if ($e > 1);
+        }
         return undef;
 }
@@ -651 +692 @@
 sub canonicalizeTitles_match
 {
-        my $word1=canonicalizeTitle(shift);
-        my $word2 =canonicalizeTitle(shift);
+        my $word1 = canonicalizeTitle(shift);
+        my $word2 = canonicalizeTitle(shift);
         my @longer;
         my @shorter;
@@ -664 +705 @@
         }
 
-#printf "got shorter: '%s', longer '%s'\n",$shorter,$longer;
         WORD: for my $word (@shorter) {
-#printf "got word: '%s'\n",$word;
                 for(my $i=0; $i < @longer; ++$i) {
                         if (forgivingMatch($longer[$i], $word)) {
@@ -799 +838 @@
 
         if ($c->{'generator-info-name'}) {
-                if ($c->{'generator-info-name'} =~ /\s?(\S+)\sv([\d\.]+)/) {
+                if ($c->{'generator-info-name'} =~ /\s?(\S+)\s[v]?([\d\.]+)/) {
                         ($proggy, $version) = ($1,$2);
                 }
@@ -1054 +1093 @@
         my @titles, my @titles_from;
         my $num_titles = 0;
+        my $channel = $m->[0]->{channel};
+        my $title_start = $m->[0]->{start_epoch};
+        my $title_duration = $m->[0]->{stop_epoch} - $m->[0]->{start_epoch};
 
         # 0. first gather title from "preferred grabber" if we have it for
@@ -1190 +1232 @@
         }
 
+        my $seen_in_primary = time;
+
+        # 6. look back through our history of programmes to see if we can fuzzy match
+        #    this title to a previously-seen title
+        for my $title_cache_key (keys %{$title_history}) {
+                my ($th_starttime, $th_duration, $th_channel, $th_grabber) = split(/,/,$title_cache_key);
+
+                # has to be on same channel
+                next if ($channel ne $th_channel);
+
+                # has to be on the same day of the week
+                next if ((int($th_starttime / 86400) % 7) != (int($title_start / 86400) % 7));
+
+                # duration has to be within 10% ($reclogic{title_history_duration_fuzzy_match_percent})
+                next if ($title_duration > ($th_duration * (1 + ($reclogic{title_history_duration_fuzzy_match_percent} / 100))));
+                next if ($title_duration < ($th_duration * (1 - ($reclogic{title_history_duration_fuzzy_match_percent} / 100))));
+
+                # start time has to be within 20 minutes ($reclogic{title_history_start_fuzzy_match_window})
+                my $th_starttime_day = $th_starttime % (7*86400);
+                my $title_start_day = $title_start % (7*86400);
+                next if ($title_start_day > ($th_starttime_day + $reclogic{title_history_start_fuzzy_match_window}));
+                next if ($title_start_day < ($th_starttime_day - $reclogic{title_history_start_fuzzy_match_window}));
+
+                # within window - check title
+                foreach my $i (0..($num_titles-1)) {
+                        # grabber has to be different
+                        next if ($th_grabber eq $titles_from[$i]);
+
+                        # check match
+                        my $match = canonicalizeTitles_match($title_history->{$title_cache_key},$titles[$i]);
+                        &log($reclogic{debug_choose_title},(sprintf "choose_title: title \"%s\" (from %s) %s title_history \"%s\" (from %s)",
+                                $titles[$i], $titles_from[$i], ($match == 1 ? "IS THE SAME AS" : "did not match"),
+                                $title_history->{$title_cache_key}, $th_grabber));
+
+                        if ($match) {
+                                # match - add it to the front of our titles list!
+                                $num_titles++;
+                                unshift(@titles, $title_history->{$title_cache_key});
+                                unshift(@titles_from, $th_grabber);
+                                $seen_in_primary = $th_starttime;
+                                last;
+                        }
+                }
+        }
+
         my $preferred_title = $titles[0];
 
-        # 6. don't create a preferred title for this unless we have met our threshold for title_xlate_table_min_alt_progs
+        # 7. don't create a preferred title for this unless we have met our threshold for title_xlate_table_min_alt_progs
         if (($num_titles-1) < $reclogic{title_xlate_table_min_alt_progs}) {
                 &log($reclogic{debug_choose_title},(sprintf 
@@ -1200 +1287 @@
         }
 
-        # 7. wasn't an alternate name
+        # 8. wasn't an alternate name
         #    add it as a preferred name
 
         my $key = lc($preferred_title);
         $title_xlate_table->{$key}->{seen_in_primary} = 1;
-        $title_xlate_table->{$key}->{last_seen_in_primary} = time;
+        $title_xlate_table->{$key}->{last_seen_in_primary} = $seen_in_primary;
         $title_xlate_table->{$key}->{translated} = 0;
         $title_xlate_table->{$key}->{translation} = $preferred_title;
@@ -1261 +1348 @@
         $newprog->{title}->[0] =        [ $title, $title_lang ];
 
+        # add this programme to our title_history
+        my $title_history_key = sprintf "%d,%d,%s,%s", 
+                $newprog->{start_epoch}, 
+                ($newprog->{stop_epoch}-$newprog->{start_epoch}), 
+                $newprog->{channel}, $m->[0]->{grabber};
+        $title_history->{$title_history_key} = $title;
+
+        # fill in programme fields and attributes
         $newprog->{datasources} = "";
         for my $i (0..($num_matching-1)) {

status

@@ -11 +11 @@
 grabber         ninemsn             0.01-r1
 grabber         yahoo7web           0.01-r1
-reconciler      reconciler_mk2      0.14
+reconciler      reconciler_mk2      0.15
 postprocessor   imdb_augment_data   0.05
 postprocessor   flag_aus_hdtv       0.10.3-r1