Changeset 73 for reconcilers/reconciler_mk2

Timestamp:

10/16/06 16:12:02 (7 years ago)

Author:

lincoln

Message:

variety of bug fixes and enhancements

(1) yahoo7widget:

• fill in extra fields correctly (cast/credits/categories/date/ratings etc)
• use the same categories translation table as what Max has in Rex
• don't output times with any timezone

(2) reconciler_mk2:

• fix some XMLTV parsing so as to output some of the more obscure fields correctly
• DateTime::Format::Strptime was actually parsing timezones when we weren't asking it to, skewing all dates. roll our own parse_xmltv_date to get around such bogosity

Files:

• reconcilers/reconciler_mk2 (modified) (14 diffs)

reconcilers/reconciler_mk2

@@ -14 +14 @@
 #    0.02  11aug06      complete rewrite, new algorithms, seperate out from shepherd
 #    0.03  15aug06      first committed
+#    0.04  17aug06      bug fix: use our own parse_xmltv_date, Date::Manip seems broken
+#                       bug fix: correctly parse some XMLTV rare fields 
 
 #
@@ -73 +75 @@
 
 my $progname = "reconciler_mk2";
-my $version = "0.03_15aug06";
+my $version = "0.04_15aug06";
 
 use LWP::UserAgent;
@@ -84 +86 @@
 use Data::Dumper;
 use Compress::Zlib;
-use DateTime::Format::Strptime;
 use Cwd;
 
@@ -171 +172 @@
 BEGIN { %amp = ( nbsp => ' ', qw{ amp & lt < gt > apos ' quot " } ) }
 
-my @strptime;
-$strptime[0] = new DateTime::Format::Strptime( pattern => "%Y%m%d%H%M %z");
-$strptime[1] = new DateTime::Format::Strptime( pattern => "%Y%m%d%H%M%S %z");
-$strptime[2] = new DateTime::Format::Strptime( pattern => "%Y%m%d%H%M");
-# $strptime[3] = new DateTime::Format::Strptime( pattern => "%Y%m%d%H%M%S");
 
 #
@@ -422 +418 @@
 
 ######################################################################################################
+
+sub parse_xmltv_date
+{
+        my $datestring = shift;
+        my @timeattr; # = localtime(time); # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
+
+        if ($datestring =~ /^(....)(..)(..)(..)(..)/) {
+                ($timeattr[5],$timeattr[4],$timeattr[3],$timeattr[2],$timeattr[1],$timeattr[0]) = (int($1)-1900,int($2)-1,int($3),int($4),int($5),0);
+                ($timeattr[6],$timeattr[7],$timeattr[8]) = (-1,-1,-1);
+                # NOTE: FIXME: we are ignoring timezone for now...
+
+                my $return_epoch = mktime(@timeattr);
+                return $return_epoch if ($return_epoch > 1);
+        }
+
+        return undef;
+}
+
+######################################################################################################
 # original alternativeTitles() from XMLTV::IMDB, this one knows about even more translations..
 
@@ -725 +740 @@
 
         # work out epoch times
-        #printf "got start '%s' stop '%s'\n",$prog->{start},$prog->{stop};
-        my $t1, my $t2;
-        foreach my $stime (@strptime) {
-                $t1 = $stime->parse_datetime($prog->{start});
-                $t2 = $stime->parse_datetime($prog->{stop});
-
-                &log($reclogic{debug_parse_time},
-                  (sprintf "  prog \"%s\" on chan \"%s\" start %s end %s, duration %d, file %s",
-                  $prog_title, $prog_chan, (strftime "%a%e%b%H:%M", localtime($t1->epoch)),
-                  (strftime "%a%e%b%H%M", localtime($t2->epoch)), ($t2->epoch - $t1->epoch), $source)) if ($t1 && $t2);
-
-                last if ($t1 && $t2);
-        }
+        my $t1 = &parse_xmltv_date($prog->{start});
+        my $t2 = &parse_xmltv_date($prog->{stop});
+
         if (!$t1 || !$t2) {
                 &log($reclogic{warn_on_invalid_time_in_prog},
@@ -747 +752 @@
         }
 
-        if (($t2->epoch - $t1->epoch) > $reclogic{max_programme_length}) {
+        &log($reclogic{debug_parse_time},
+          (sprintf "  prog \"%s\" on chan \"%s\" start %s end %s, duration %d, file %s",
+          $prog_title, $prog_chan, (strftime "%a%e%b%H:%M", localtime($t1)),
+          (strftime "%a%e%b%H%M", localtime($t2)), ($t2 - $t1), $source)) if ($t1 && $t2);
+#printf "got start '%s', turned it into start_epoch '%s'\n",$prog->{start},(strftime "%a %e %b %H %M",localtime($t1));
+
+        if (($t2 - $t1) > $reclogic{max_programme_length}) {
                 &log($reclogic{warn_on_invalid_time_in_prog},
                   (sprintf "WARNING: programme '%s' on channel '%s' from %s had a programme duration (%d) that exceeded max_programme_length (%d): start '%s' stop '%s'; ignored.\n",
-                  $prog_title, $prog_chan, $datafile, ($t2->epoch - $t1->epoch),  $reclogic{max_programme_length},
+                  $prog_title, $prog_chan, $datafile, ($t2-$t1),  $reclogic{max_programme_length},
                   ($prog->{start} ? $prog->{start} : "undef"), ($prog->{stop} ? $prog->{stop} : "undef")));
                 $stats{bad_programme_duration_too_long}++;
@@ -756 +767 @@
         }
 
-        $prog->{start_epoch} = $t1->epoch;
-        $prog->{stop_epoch} = $t2->epoch;
+        $prog->{start_epoch} = $t1;
+        $prog->{stop_epoch} = $t2;
         $prog->{grabber} = $source;
         $prog->{grabber_num} = ($in->{num_datafiles}-1);
-        my $prog_key = sprintf "%d:%d",$t1->epoch,$t2->epoch;
+        my $prog_key = sprintf "%d:%d",$t1,$t2;
 
         # if there is a ": " in title and no subtitle, split title into "title: subtitle"
@@ -795 +806 @@
                 &log($warn,(sprintf "WARNING: file contained more than one programme in same timeslot: channel '%s' start %s, stop %s, programs \"%s\" and \"%s\"; ignored.",
                         $prog_chan,
-                        (strftime "%a %e %b %H:%M",localtime($t1->epoch)),
-                        (strftime "%H:%M",localtime($t2->epoch)),
+                        (strftime "%a %e %b %H:%M",localtime($t1)),
+                        (strftime "%H:%M",localtime($t2)),
                         $prog_title,
                         $in->{dupcheck}->{$source}->{$prog_chan}->{$prog_key}));
@@ -924 +935 @@
 
                 # (2) merge in attributes which are simple arrays
-                foreach my $field ('url', 'star-rating') {
+                foreach my $field ('url', 'star-rating', 'premiere', 'last-chance', 'language', 'orig-language') {
                         if (defined $m->[$i]->{$field}) {
                                 my %existing_hash;
@@ -950 +961 @@
 
                 # (3) merge in attributes which are [val,lang] pairs, [val,val] or [val,val,val]
-                foreach my $field ('title', 'sub-title', 'desc', 'language', 'orig-language',
-                  'category', 'country', 'premiere', 'last-chance', 'episode-num', 'rating') {
+                foreach my $field ('title', 'sub-title', 'desc', 'category', 'country', 'episode-num', 'rating') {
                         my $num_added = 0;
                         my %lang_added;
@@ -1032 +1042 @@
 
                 # (5) merge in attributes which are by name with no values, or values we don't care to interpret
-                foreach my $field ('new', 'subtitles') {
+                foreach my $field ('new', 'subtitles', 'previously-shown') {
                         if ((!defined $newprog->{$field}) && (defined $m->[$i]->{$field})) {
                                 $newprog->{$field} = $m->[$i]->{$field};
@@ -1039 +1049 @@
 
                 # (6) merge in attributes which are {hash}->{hash}=val
-                foreach my $field ('video', 'audio', 'subtitles') {
+                foreach my $field ('video', 'audio') {
                         my %entries;
 
@@ -1064 +1074 @@
                 # ignored attributes:
                 #   icon
-                #   previously-shown
         }