Changeset 105

Timestamp:

10/19/06 23:42:09 (7 years ago)

Author:

lincoln

Message:

much improved title-preference rewriting + logs that go with it

Files:

• reconcilers/reconciler_mk2 (modified) (22 diffs)
• status (modified) (1 diff)

reconcilers/reconciler_mk2

@@ -19 +19 @@
 #    0.06  17aug06      enhancement: name mapping based on previously learnt names
 #    0.07  19aug06      turn down debugging
+#    0.08  20aug06      improved preference-title rewriter, more debugging to logfile
 
 #
@@ -71 +72 @@
 #       duplicate checking:             $in->{dupcheck}->{$source}->{$prog_chan}->{$prog_key}
 #       channel list:                   $in->{channels}->{$chan}->{$lang} = $desc
-
+#
+#       used translations               $in->{used_translations}->{$xlate_to}->{num}=count
+#                                       $in->{used_translations}->{$xlate_to}->{from}->{$xlate_from}=count
+
+# history of title preferences uses the following (cache-based) data:
+#   $title_xlate_table->{"$primary_title"}->{seen_in_primary} = count
+#   $title_xlate_table->{"$primary_title"}->{last_seen_in_primary} = time
+#   $title_xlate_table->{"$primary_title"}->{translated} = count
+#   $title_xlate_table->{"$primary_title"}->{translation} = title
+#   $title_xlate_table->{"$primary_title"}->{alternate}->{"$alt_title"}->{secondary}->{"$source"} = count
+#   $title_xlate_table->{"$primary_title"}->{alternate}->{"$alt_title"}->{last_seen_in_secondary} = time
+#   $title_xlate_table->{"$primary_title"}->{same}->{"$source"} = count
 
 use strict;
@@ -78 +90 @@
 
 my $progname = "reconciler_mk2";
-my $version = "0.07_19aug06";
+my $version = "0.08_20aug06";
 
 use LWP::UserAgent;
@@ -127 +139 @@
 $reclogic{max_programme_length} = (8 * 60 * 60); # 8 hours
 
+# do store a preference title, even if we don't have any alternatives
+$reclogic{title_xlate_table_min_alt_progs} = 0;
 
 #
@@ -176 +190 @@
 my $setting_override;
 my %cli_override;
-my $title_map_table;
+my $title_xlate_table;  # cached
 
 my %amp;
@@ -188 +202 @@
 my $opt = { };
 $opt->{output_file} =           cwd()."/output.xmltv";
-$opt->{config_file} =           cwd()."/".$progname.".conf";
+$opt->{config_file} =           cwd()."/".$progname.".config";
 $opt->{log_file} =              cwd()."/".$progname.".log";
 $opt->{alt_title_file} =        cwd()."/".$progname.".alt_title.log";
@@ -347 +361 @@
         open(CONF, ">$opt->{config_file}") || die "cannot write to $opt->{config_file}: $!";
         print CONF Data::Dumper->Dump(
-                [$setting_override,  $title_map_table  ],
-                ["setting_override", "title_map_table" ]);
+                [$setting_override,  $title_xlate_table  ],
+                ["setting_override", "title_xlate_table" ]);
         close CONF;
         &log(1,(sprintf "updated configuration file %s.\n",$opt->{config_file}));
@@ -489 +503 @@
         my ($log_level,$entry) = @_;
         printf STDERR "%s [%d] %s\n",$progname, time,$entry if ($log_level);
-        printf LOG_FILE "%s [%d] %s\n",$progname, time,$entry if (($log_level) && (!$opt->{nolog}));
+        #printf LOG_FILE "%s [%d] %s\n",$progname, time,$entry if (($log_level) && (!$opt->{nolog}));
+        printf LOG_FILE "%s [%d] %s\n",$progname, time,$entry unless ($opt->{nolog});
 }
 
@@ -553 +568 @@
         printf ALT_TITLE_FILE "(1) The following list shows title translations that were used:\n\n";
 
-        printf ALT_TITLE_FILE "  Preferred Title (translated to)   Translated From                  Seen\n";
-        printf ALT_TITLE_FILE "  --------------------------------  ------------------------------  -----\n";
-        my $used = $in->{used_translations};
-        foreach my $key (sort { $used->{$b}->{num} <=> $used->{$a}->{num} } keys %{$used}) {
-                my $used2 = $used->{$key}->{from};
-                foreach my $key2 (sort { $used2->{$a} <=> $used2->{$b} } keys %{$used2}) {
-                        printf ALT_TITLE_FILE "  %-32s  %-30s  %5d\n", $key, $key2, $used2->{$key2};
+        printf ALT_TITLE_FILE "  Times Preferred Title (translated to)  Translated from                Times\n";
+        printf ALT_TITLE_FILE "  ----- -------------------------------- ------------------------------ -----\n";
+        my $u = $in->{used_translations};
+        foreach my $xlate_to (sort { $u->{$b}->{num} <=> $u->{$a}->{num} } keys %{$u}) {
+                printf ALT_TITLE_FILE "  %5d %-32s", $u->{$xlate_to}->{num},$xlate_to;
+
+                my $u2 = $u->{$xlate_to}->{from};
+                my $num = 0;
+                foreach my $xlate_from (sort { $u2->{$a} <=> $u2->{$b} } keys %{$u2}) {
+                        printf ALT_TITLE_FILE "%s %-30s %5d\n",
+                          ($num > 0 ? "                                        " : ""),
+                          $xlate_from, $u2->{$xlate_from};
+                        $num++;
                 }
         }
@@ -565 +586 @@
         printf ALT_TITLE_FILE "\n==========================================================================================================\n\n";
         printf ALT_TITLE_FILE "(2) The following table lists ALL the translations we have stored:\n\n";
-        printf ALT_TITLE_FILE " Preferred title (xlate to)         Translated From                    Last used (total) Last seen (total)\n";
-        printf ALT_TITLE_FILE " ---------------------------------- ---------------------------------- ----------------- -----------------\n";
-
-        foreach my $key (sort keys %{$title_map_table}) {
-                my $xlate_to = $title_map_table->{$key};
-                foreach my $key2 (sort keys %{$xlate_to}) {
-                        printf ALT_TITLE_FILE " %-34s %-34s %-10s %6d %-10s %6d\n",
-                                substr($key2,0,34), substr($key,0,34),
-                                ($title_map_table->{$key}->{$key2}->{last_translated} ?
-                                  (strftime "%a%e%b%y",localtime($title_map_table->{$key}->{$key2}->{last_translated})) : "never"),
-                                ($title_map_table->{$key}->{$key2}->{times_translated} ?
-                                  $title_map_table->{$key}->{$key2}->{times_translated} : 0),
-                                ($title_map_table->{$key}->{$key2}->{last_seen} ? 
-                                  (strftime "%a%e%b%y",localtime($title_map_table->{$key}->{$key2}->{last_seen})) : "never"),
-                                ($title_map_table->{$key}->{$key2}->{times_seen} ? 
-                                  $title_map_table->{$key}->{$key2}->{times_seen} : 0);
-                }
+        printf ALT_TITLE_FILE "+Preferred title (xlate to)          xlate_  preferred title seen in source (times)\n";
+        printf ALT_TITLE_FILE "-Alternate title (xlate from)         times  alternate title seen in source (times)\n";
+        printf ALT_TITLE_FILE " ---------------------------------- -------  --------------------------------------------------------------\n";
+
+        foreach my $xlate_to (sort { $title_xlate_table->{$b}->{translated} <=> $title_xlate_table->{$a}->{translated} } keys %{$title_xlate_table}) {
+                printf ALT_TITLE_FILE "+%-34s %7d ",
+                  substr("\"".$title_xlate_table->{$xlate_to}->{translation}."\"",0,34),
+                  $title_xlate_table->{$xlate_to}->{translated};
+
+                foreach my $source (sort { $title_xlate_table->{$xlate_to}->{same}->{$b} <=> $title_xlate_table->{$xlate_to}->{same}->{$a} }
+                  keys %{($title_xlate_table->{$xlate_to}->{same})}) {
+                        printf ALT_TITLE_FILE " %s(%d)", $source, $title_xlate_table->{$xlate_to}->{same}->{$source};
+                }
+                printf ALT_TITLE_FILE "\n";
+
+                foreach my $xlate_from (sort keys %{($title_xlate_table->{$xlate_to}->{alternate})}) {
+                        my $last_xlated = (time - $title_xlate_table->{$xlate_to}->{alternate}->{$xlate_from}->{last_seen_in_secondary});
+                        my $last_xlated_desc;
+                        if ($last_xlated < (24*60*60)) {
+                                $last_xlated_desc = sprintf "-%dhrs",int($last_xlated/(24*60*60));
+                        } elsif ($last_xlated < (7*24*60*60)) {
+                                $last_xlated_desc = sprintf "-%ddays",int($last_xlated/(24*60*60));
+                        } else {
+                                $last_xlated_desc = sprintf "-%dwks",int($last_xlated/(7*24*60*60));
+                        }
+                        
+                        printf ALT_TITLE_FILE "-%-34s %7s ",substr("\"".$xlate_from."\"",0,34),$last_xlated_desc;
+
+                        foreach my $source (sort { $title_xlate_table->{$xlate_to}->{alternate}->{$xlate_from}->{secondary}->{$a} <=>
+                          $title_xlate_table->{$xlate_to}->{alternate}->{$xlate_from}->{secondary}->{$b} }
+                          keys %{($title_xlate_table->{$xlate_to}->{alternate}->{$xlate_from}->{secondary})}) {
+                                printf ALT_TITLE_FILE " %s(%d)", $source, $title_xlate_table->{$xlate_to}->{alternate}->{$xlate_from}->{secondary}->{$source};
+                        }
+                        printf ALT_TITLE_FILE "\n";
+                }
+                printf ALT_TITLE_FILE "\n";
         }
 
@@ -1064 +1104 @@
 
 ######################################################################################################
+# all logic associated with choosing the preferred title of a programme
+# (uses past-seen choices to set preferences)
+
+sub choose_title
+{
+        my ($num_matching,$m) = @_;
+        my @titles, my @titles_from;
+        my $num_titles = 0;
+
+        # 1. gather a list of all titles we have for our preferred language
+        #    there are stored in the @titles array in our "preferred grabber" order
+        for my $i (0..($num_matching-1)) {
+                if (defined $m->[$i]->{title}) {
+                        foreach my $found_t (@{($m->[$i]->{title})}) {
+                                my $t = $found_t->[0];
+                                my $l = $found_t->[1];
+
+                                if ((!defined $l) || (lc($l) eq lc($opt->{lang}))) {
+                                        $titles[$num_titles] = $t;
+                                        $titles_from[$num_titles] = $m->[$i]->{grabber};
+                                        $num_titles++;
+                                }
+                        }
+                }
+        }
+
+        # 2. no titles in our language? return with the first title we saw
+        if ($num_titles == 0) {
+                my $title = $m->[0]->{title}->[0]->[0];
+                my $lang = $m->[0]->{title}->[0]->[1];
+
+                &log($reclogic{debug_choose_title},(sprintf 
+                  "choose_title: no titles in lang %s found; using first title \"%s\" in lang %s",
+                  $opt->{lang}, $title, (defined $lang ? $lang : "(undef)")));
+                return ($title, $lang);
+        }
+
+        # 3. do we already have any titles in our list here in our 'preferred title' list?
+        my $found_index = -1;
+        foreach my $i (0..$num_titles) {
+                if (defined $title_xlate_table->{(lc($titles[$i]))}) {
+                        $found_index = $i;
+                        last;
+                }
+        }
+
+        # 4. found our preferred title.  use that.
+        if ($found_index != -1) {
+                my $key = lc($titles[$found_index]);
+                my $title = $title_xlate_table->{$key}->{translation};
+
+                &log($reclogic{debug_choose_title},(sprintf 
+                  "choose_title: existing preferred title \"%s\" matched with title \"%s\" from grabber %s (index %d)",
+                  $title, $titles[$found_index], $titles_from[$found_index], $found_index));
+
+                # update stats on use
+                $title_xlate_table->{$key}->{seen_in_primary}++;
+                $title_xlate_table->{$key}->{last_seen_in_primary} = time;
+
+                # update 'seen' stats for alternatives
+                foreach my $i (0..($num_titles-1)) {
+                        my $key2 = $titles[$i]; # don't actually store in lower-case..
+                        my $source = $titles_from[$i];
+
+                        if (lc($key) eq lc($key2)) {
+                                # store where we have seen the same title
+                                $title_xlate_table->{$key}->{same}->{$source}++;
+                        } else {
+                                # store all alternatives for this title
+                                $title_xlate_table->{$key}->{alternate}->{$key2}->{secondary}->{$source}++;
+                                $title_xlate_table->{$key}->{alternate}->{$key2}->{last_seen_in_secondary} = time;
+                        }
+                }
+
+                return ($title, $opt->{lang});
+        }
+
+        # 5. didn't find this title in our preferred title list.
+        #    see if it is an alternate name
+        foreach my $i (0..($num_titles-1)) {
+                my $title = $titles[$i]; # don't actually convert to lowercase..
+
+                # FUTURE MAYBE: this is a linear search.
+                # if it grows too much consider a schema change to make it better
+
+                foreach my $key (keys %{($title_xlate_table)}) {
+                        foreach my $key2 (keys %{($title_xlate_table->{$key}->{alternate})}) {
+                                if (lc($key2) eq lc($title)) {
+                                        # matched alternate!
+                                        $title_xlate_table->{$key}->{translated}++;
+                                        my $xlated_to = $title_xlate_table->{$key}->{translation};
+                                        my $xlated_from = $titles[$i];
+                                        $in->{used_translations}->{$xlated_to}->{num}++;
+                                        $in->{used_translations}->{$xlated_to}->{from}->{$xlated_from}++;
+
+                                        &log($reclogic{debug_choose_title},(sprintf 
+                                          "choose_title: title \"%s\" (idx %d grabber %s) matched previously-seen alternate title for \"%s\"",
+                                          $xlated_from, $i, $titles_from[$i], $xlated_to));
+
+                                        # update 'seen' stats for alternatives
+                                        foreach my $j (0..($num_titles-1)) {
+                                                my $key2a = $titles[$j]; # don't store in lowercase
+                                                my $source = $titles_from[$j];
+
+                                                if ($key eq lc($key2a)) {
+                                                        # store where we have seen the same title
+                                                        $title_xlate_table->{$key}->{same}->{$source}++;
+                                                } else {
+                                                        # store all alternatives for this title
+                                                        $title_xlate_table->{$key}->{alternate}->{$key2a}->{secondary}->{$source}++;
+                                                        $title_xlate_table->{$key}->{alternate}->{$key2a}->{last_seen_in_secondary} = time;
+                                                }
+                                        }
+
+                                        # return primary translation for this found alternative
+                                        return ($xlated_to, $opt->{lang});
+                                }
+                        }
+                }
+        }
+
+        my $preferred_title = $titles[0];
+
+        # 6. don't create a preferred title for this unless we have met our threshold for title_xlate_table_min_alt_progs
+        if (($num_titles-1) < $reclogic{title_xlate_table_min_alt_progs}) {
+                &log($reclogic{debug_choose_title},(sprintf 
+                  "choose_title: title \"%s\" not added as a preferred title due to title_xlate_table_min_alt_progs (%d)",
+                  $preferred_title, $reclogic{title_xlate_table_min_alt_progs}));
+                return ($preferred_title, $opt->{lang});
+        }
+
+        # 7. wasn't an alternate name
+        #    add it as a preferred name
+
+        my $key = lc($preferred_title);
+        $title_xlate_table->{$key}->{seen_in_primary} = 1;
+        $title_xlate_table->{$key}->{last_seen_in_primary} = time;
+        $title_xlate_table->{$key}->{translated} = 0;
+        $title_xlate_table->{$key}->{translation} = $preferred_title;
+        $title_xlate_table->{$key}->{same}->{$titles_from[0]} = 1;
+
+        foreach my $i (1..($num_titles-1)) {
+                my $key2 = $titles[$i];
+                my $source = $titles_from[$i];
+
+                if (lc($key2) eq $key) {
+                        $title_xlate_table->{$key}->{same}->{$source}++;
+                } else {
+                        $title_xlate_table->{$key}->{alternate}->{$key2}->{secondary}->{$source} = 1;
+                        $title_xlate_table->{$key}->{alternate}->{$key2}->{last_seen_in_secondary} = time;
+                }
+        }
+
+        &log($reclogic{debug_choose_title},(sprintf 
+          "choose_title: added title \"%s\" as a preferred title (grabber %s)",
+          $preferred_title, $titles_from[0]));
+
+        return ($preferred_title, $opt->{lang});
+}
+
+######################################################################################################
 # insert a programme into our reconciled list
 #  - if fed multiple programmes for the one slot, choose attributes from all of them
@@ -1069 +1270 @@
 sub add_one_programme
 {
-        my($chan_id,$num_matching,$m) = @_;
+        my($chan_id,$title,$title_lang,$num_matching,$m) = @_;
         &log($reclogic{debug_add_logic},
           (sprintf "add_one_programme: adding programme %s-%s \"%s\" with data from %d programmes",
           (strftime "%a%e%b%H:%M",localtime(($m->[0]->{start_epoch}))),
           (strftime "%a%e%b%H:%M",localtime(($m->[0]->{stop_epoch}))),
-          $m->[0]->{title}->[0]->[0], $num_matching));
+          $title, $num_matching));
 
         my $newprog;
         my $chan = $m->[0]->{channel};
         my $starttime = $m->[0]->{start_epoch};
+        my $have_xlated_title = 0;
 
         # always take start/stop/channel from first choice
@@ -1092 +1294 @@
         $newprog->{stop_epoch} =        $m->[0]->{stop_epoch};
         $newprog->{channel} =           $chan;
+        $newprog->{title}->[0] =        [ $title, $title_lang ];
 
         $newprog->{datasources} = "";
@@ -1099 +1302 @@
                 # (1) merge in attributes which are simple scalars
                 foreach my $field ('date', 'length') {
-                        $newprog->{$field} = $m->[$i]->{$field}
-                          if ((!defined $newprog->{$field}) && (defined $m->[$i]->{$field}) && ($m->[$i]->{$field} ne ""));
+                        if ((!defined $newprog->{$field}) &&
+                            (defined $m->[$i]->{$field}) && 
+                            ($m->[$i]->{$field} ne "")) {
+                                $newprog->{$field} = $m->[$i]->{$field};
+                                &log($reclogic{debug_add_logic},
+                                  (sprintf " - added field %s from #%d (%s) value %s",
+                                  $field,$m->[$i]->{grabber_num},$m->[$i]->{grabber},$newprog->{$field}));
+                        }
                 }
 
@@ -1124 +1333 @@
                                                 $newprog->{$field}->[$num_existing] = $entry;
                                                 $num_existing++;
+
+                                                &log($reclogic{debug_add_logic},
+                                                  (sprintf " - added field %s from #%d (%s) value %s",
+                                                  $field,$m->[$i]->{grabber_num},$m->[$i]->{grabber},$entry));
                                         }
                                 }
@@ -1130 +1343 @@
 
                 # (3) merge in attributes which are [val,lang] pairs, [val,val] or [val,val,val]
+                #     note: while 'title' is here, this is simply to pick up titles in any
+                #     additional languages; the primary (preferred) title has already been
+                #     set via the choose_title() routine
                 foreach my $field ('title', 'sub-title', 'desc', 'category', 'country', 'episode-num', 'rating') {
                         my $num_added = 0;
@@ -1164 +1380 @@
                                                 $lang_added{$lang} = $val;
 
-                                                # title is special - we write out alternate title if we have one
-                                                if (($field eq "title") && ($lang eq $opt->{lang})) {
-                                                        if (defined $title_map_table->{$val}) {
-                                                                # got a translation to make!
-                                                                # if we have multiple to choose from, choose the one
-                                                                # used the most
-
-                                                                my $translations = $title_map_table->{$val};
-                                                                my @translation_list = sort { choose_best_translation($val,$a,$b) } keys %{$translations};
-                                                                my $chosen_xlate = $translation_list[0];
-
-                                                                if (lc($chosen_xlate) ne lc($val)) {
-                                                                        $title_map_table->{$val}->{$chosen_xlate}->{times_translated}++;
-                                                                        $title_map_table->{$val}->{$chosen_xlate}->{last_translated} = time;
-                                                                        $in->{used_translations}->{$chosen_xlate}->{from}->{$val}++;
-                                                                        $in->{used_translations}->{$chosen_xlate}->{num}++;
-
-                                                                        &log($reclogic{debug_add_logic_name_xlate},
-                                                                          (sprintf "add_one_programme: chose translation '%s' for title '%s'",
-                                                                          $chosen_xlate, $val));
-
-                                                                        $val = $chosen_xlate;
-                                                                        $stats{used_translated_title}++;
-                                                                }
-                                                        }
-                                                }
                                                 $newprog->{$field}->[$num_added]->[0] = $val;
+
+                                                &log($reclogic{debug_add_logic},
+                                                  (sprintf " - added field %s(%s) from #%d (%s) value %s",
+                                                  $field,($lang ne "BLANK" ? $lang : ""),$m->[$i]->{grabber_num},$m->[$i]->{grabber},$val));
 
                                                 if ($lang eq "BLANK") {
@@ -1231 +1425 @@
                                                         $entries{$entry} = 1;
                                                         $num_added++;
+
+                                                        &log($reclogic{debug_add_logic},
+                                                          (sprintf " - added field %s/%s from #%d (%s) value %s",
+                                                          $field,$field2,$m->[$i]->{grabber_num},$m->[$i]->{grabber},$entry));
                                                 }
                                         }
@@ -1241 +1439 @@
                         if ((!defined $newprog->{$field}) && (defined $m->[$i]->{$field})) {
                                 $newprog->{$field} = $m->[$i]->{$field};
+
+                                &log($reclogic{debug_add_logic},
+                                  (sprintf " - added field %s from #%d (%s)",
+                                  $field,$m->[$i]->{grabber_num},$m->[$i]->{grabber}));
                         }
                 }
@@ -1263 +1465 @@
                                                 $newprog->{$field}->{$entry} = $m->[$i]->{$field}->{$entry};
                                                 $entries{$entry} = $m->[$i]->{$field}->{$entry};
+
+                                                &log($reclogic{debug_add_logic},
+                                                  (sprintf " - added field %s from #%d (%s) value %s",
+                                                  $field,$m->[$i]->{grabber_num},$m->[$i]->{grabber},$entries{$entry}));
                                         }
                                 }
@@ -1279 +1485 @@
                 $in->{total_progs_out}->{($newprog->{channel})}++;
         }
-}
-
-######################################################################################################
-# logic for choosing the best translation
-# called from sort of $title_map_table->{}
-
-# choose "most translated"
-# if zero, then choose "most seen"
-
-sub choose_best_translation
-{
-        my ($v,$a,$b) = @_;
-        my $most_translated_result = ($title_map_table->{$v}->{$b}->{times_translated} <=> $title_map_table->{$v}->{$a}->{times_translated});
-        return $most_translated_result if ($most_translated_result != 0);
-        return ($title_map_table->{$v}->{$b}->{times_seen} <=> $title_map_table->{$v}->{$a}->{times_seen});
 }
 
@@ -1573 +1764 @@
                                   $this_title, $i, $matching_progs->[$num_matching_progs]->{grabber}));
 
-                                # store details of this in our mappings table if titles differ
-                                if (lc($primary_title) ne lc($this_title)) {
-                                        if ($title_map_table->{$this_title}->{$primary_title}->{times_seen}) {
-                                                $stats{already_seen_title_translations}++;
-                                        } else {
-                                                $stats{added_title_translation}++;
-                                                $title_map_table->{$this_title}->{$primary_title}->{times_translated} = 0;
-                                                $title_map_table->{$this_title}->{$primary_title}->{times_seen} = 0;
-                                                &log($reclogic{debug_find_prog_to_add},
-                                                  (sprintf "find_prog_to_add:  - saving title translation: preferred \"%s\", alternate \"%s\"",
-                                                  $primary_title, $this_title));
-                                        }
-                                        $title_map_table->{$this_title}->{$primary_title}->{times_seen}++;
-                                        $title_map_table->{$this_title}->{$primary_title}->{last_seen} = time;
-                                }
-
                                 $num_matching_progs++;
                         }
                 }
         }
-        &add_one_programme($chan_id,$num_matching_progs,$matching_progs);
+
+        my ($title, $title_lang) = &choose_title($num_matching_progs,$matching_progs);
+        &add_one_programme($chan_id,$title,$title_lang,$num_matching_progs,$matching_progs);
 
         # remove all programming data for this channel which overlaps with this (inserted) programme

status

@@ -7 +7 @@
 grabber:jrobbo:0.03-r4
 grabber:d1:0.6.2.3-r4
-reconciler:reconciler_mk2:0.07
+reconciler:reconciler_mk2:0.08
 postprocessor:imdb_augment_data:0.03