Changeset 950

Timestamp:

10/17/07 23:51:41 (6 years ago)

Author:

max

Message:

update branch: svn merge -r 941:949 http://svn.whuffy.com/shepherd/trunk .

Location:

branches/predictive

Files:

• applications/shepherd (modified) (7 diffs)
• grabbers/abc_website.conf (modified) (1 diff)
• grabbers/acctv_website (modified) (4 diffs)
• grabbers/citysearch (modified) (9 diffs)
• grabbers/oztivo (modified) (3 diffs)
• grabbers/rex (modified) (14 diffs)
• grabbers/southerncross_website (modified) (7 diffs)
• grabbers/yahoo7widget (modified) (4 diffs)
• reconcilers/reconciler_mk2 (modified) (3 diffs)
• references/Shepherd/Common.pm (modified) (2 diffs)
• status (modified) (1 diff)

branches/predictive/applications/shepherd

@@ -2 +2 @@
 
 my $progname = 'shepherd';
-my $version = '1.2.24';
+my $version = '2.0.0';
 
 # tv_grab_au
@@ -1366 +1366 @@
     $display_output = 1 if (!defined $display_output);
     $progtype = $components->{$component}->{type} unless ($progtype);
+    if ($components->{$component}->{default_cmdline})
+    {
+        my $parameters = $components->{$component}->{default_cmdline};
+        $parameters =~ s/:/ /g;
+        $prog .= " $parameters";
+    }
 
     my $prog_output = "";
@@ -1817 +1823 @@
     my $arg = join(":",@args);
 
-    if ($component eq $progname) {
-        if ((!defined $arg) || ($arg eq "")) {
-            delete $components->{$progname}->{default_cmdline};
-            &log("\ncleared default options for $progname\n");
-        } else {
-            $components->{$progname}->{default_cmdline} = "--".join(":--",@args);
-            &log("\nset default options for $progname to: ".$components->{$progname}->{default_cmdline}."\n");
-        }
-        write_config_file();
-        return;
-    }
-
     if ((!defined $arg) || ($arg eq "")) {
-        &log("\nNothing to set: you need to use this as --component-set (component):(settings)\n");
-        return;
-    }
-
-    my $opt_set = query_config($component, 'option_set');
-    if (!defined $opt_set) {
-        &log("\nComponent $component does not have any tunable parameters.\n");
-        return;
-    }
-
-    my ($result,$resultmsg,$test_output) = call_prog($component,query_filename($component,$components->{$component}->{type})." $opt_set $arg"." 2>&1",1,1,1);
+        delete $components->{$component}->{default_cmdline};
+        &log("\nCleared default options for $component.\n");
+    } else {
+        $components->{$component}->{default_cmdline} = "--".join(":--",@args);
+        &log("\nSet default options for $component to: --".join(" --",@args)."\n");
+    }
+    &write_config_file;
 }
 
@@ -4146 +4136 @@
 
     # explicitly track unparsable dates, excessive durations, etc
-    foreach ( qw( programmes total_duration progs_with_invalid_date progs_too_long progs_too_short progs_with_unknown channel progs_outside_window progs_optional ))
+    foreach ( qw( programmes total_duration progs_with_invalid_date progs_too_long progs_too_short progs_with_unknown channel progs_outside_window progs_optional progs_tba))
     {
         $this_plugin->{$_} = 0;
@@ -4212 +4202 @@
             }
 
+            # Don't count shows that are simply 'To Be Advised'
+            # These will be dropped by the reconciler
+            if ($prog->{title}->[0]->[0] =~ /^to be advised$/i
+                    or
+                $prog->{title}->[0]->[0] =~ /^tba$/i)
+            {
+                $this_plugin->{progs_tba}++;
+                next;
+            }
+
             # store plugin-specific stats
             $this_plugin->{programmes}++;
@@ -4223 +4223 @@
             # only analyze / check against policy if its a non optional channel
             if (defined $channels->{$ch}) {
-                # store channel-specific stats
-                $channel_data->{$ch}->{programmes}++;
-                $channel_data->{$ch}->{total_duration} += $this_duration;
 
                 # programme is outside the timeslots we are interested in.
@@ -4233 +4230 @@
                     next;
                 }
+
+                # store channel-specific stats
+                $channel_data->{$ch}->{programmes}++;
+                $channel_data->{$ch}->{total_duration} += $this_duration;
 
                 # store timeslot info

branches/predictive/grabbers/abc_website.conf

@@ -6 +6 @@
             'regions' => '',
             'option_ready' => '--version',
-            'option_set' => '--set',
             'desc' => 'guide data for ABC only from http://www.abc.net.au/tv/guide/netw/',
             'max_days' => '14',

branches/predictive/grabbers/acctv_website

@@ -23 +23 @@
 my %stats;
 my $o;
-Shepherd::Common::program_begin(\$o, "acctv_website", "0.02", \%stats);
+Shepherd::Common::program_begin(\$o, "acctv_website", "0.03", \%stats);
 
 my ($channels, $opt_channels, $gaps) = Shepherd::Common::read_channels($o, ("ACC"));
@@ -232 +232 @@
                         $prog->{$_} = $details_cache->{$cache_key}->{$_};
                 }
-                if (defined $prog->{length}) {
-                        $prog->{stop} = $prog->{start} + $prog->{length};
-                        delete $prog->{length};
-                }
+                $prog->{stop} = $prog->{start} + $prog->{length} if (defined $prog->{length});
                 $stats{cached_used}++;
                 return $prog;
@@ -289 +286 @@
                 if ($header eq 'Length') {
                         if ($content !~ /^(\d+) /) { # length
-                                Shepherd::Common::log("skipped detail length : " . $td->as_HTML()) if $o->{debug};
+                                Shepherd::Common::log("skipped detail length : " . $td->as_HTML());
                                 $td = shift @tds || last;
                                 $header = $td->as_trimmed_text();
+                                $header =~ s/[^\x20-\x7f\x0a]//g; # remove  
                                 next;
                         }
                         my $length = ($1 < 3 ? ($1*60*60) : ($1*60));
                         $prog->{length} = $length;
+                        $prog->{stop} = $prog->{start} + $length;
                         Shepherd::Common::log("set length     : $length") if ($o->{debug});
                 } elsif ($header eq 'Genre') {
@@ -328 +327 @@
         }
 
-        if (defined $prog->{length}) {
-                $prog->{stop} = $prog->{start} + $prog->{length};
-                delete $prog->{length};
-        }
-
         return $prog;
 }

branches/predictive/grabbers/citysearch

@@ -4 +4 @@
 #
 
-my $version = '2.0.2';
+my $version = '2.0.3';
 
 use strict;
@@ -336 +336 @@
 
     $div = $tree->look_down(_tag => 'div', class => 'contentDetails');
-    my %video;
+    my (%video, $category, %type);
     foreach my $tr ($div->look_down(_tag => 'tr'))
     {
@@ -343 +343 @@
             if ($1 eq 'Type')
             {
-                push (@{$show->{category}}, $2) unless ($2 eq 'N/A');
+                $category = &strip_whitespace($2);
             }
             elsif ($1 eq 'Country')
@@ -349 +349 @@
                 $show->{country} = $2;
             }
+            elsif ($1 eq 'Language')
+            {
+                $show->{language} = $2;
+            }
             elsif ($1 eq 'Cast')
             {
@@ -356 +360 @@
                 }
             }
+            elsif ($1 eq 'Director')
+            {
+                foreach (split /, /, $2)
+                {
+                    push @{$show->{credits}{director}}, &strip_whitespace($_);
+                }
+            }
+            elsif ($1 eq 'Writer') # unseen
+            {
+                foreach (split /, /, $2)
+                {
+                    push @{$show->{credits}{writer}}, &strip_whitespace($_);
+                }
+            }
             elsif ($1 eq 'Duration')
             {
-                if (!$show->{stop} and $2 =~ /(\d+) minutes/)
-                {
-                    $show->{stop} = $show->{start} + ($1 * 60);
-                    &log("Filled in stop time! $1 minutes.") if ($debug);
+                if ($2 =~ /(\d+) min/)
+                {
+                    $show->{length} = $1 * 60;
+                    if (!$show->{stop}) {
+                        $show->{stop} = $show->{start} + ($1 * 60);
+                        &log("Filled in stop time! $1 minutes.") if ($debug);
+                    }
                 }
             }
@@ -388 +409 @@
                     {
                         $show->{'premiere'} = [ $info ];
+                        $type{premiere} = 1;
                     }
                     elsif ($info eq 'Live')
                     {
-                        push @{$show->{category}}, $info;
+                        $type{live} = 1;
+                    }
+                    elsif ($info eq 'Final' || $info eq 'Finale') # unseen
+                    {
+                        $type{final} = 1;
+                    }
+                    elsif ($info eq 'Return') # unseen
+                    {
+                        $type{return} = 1;
                     }
                     elsif ($info eq 'Repeat')
@@ -399 +429 @@
                     elsif ($info eq 'Movie')
                     {
-                        push @{$show->{category}}, 'movie';
+                        $type{movie} = 1;
                     }
                     else
@@ -430 +460 @@
     }
     $show->{video} = { %video } if (%video);
+    $show->{category} = [ &Shepherd::Common::generate_category(
+        $show->{title}, $category, %type) ];
 
     $tree->delete;
@@ -489 +521 @@
                 $p{$field} = [[ $p{$field}, $lang ]] if ($p{$field});
             }
+            $p{language} = [ $p{language}, $lang ] if ($p{language});
             $p{start} = &POSIX::strftime("%Y%m%d%H%M", localtime($p{start}));
             $p{stop} = &POSIX::strftime("%Y%m%d%H%M", localtime($p{stop}));
             $p{rating} = [[ $p{rating}, 'ABA', undef ]] if ($p{rating});
-            if ($p{category})
+            if ($p{category} && ref($p{category}) eq "ARRAY"
+                    && $p{category}[0] && ref($p{category}[0]) ne "ARRAY") # obsolete 14/10/2007
             {
                 foreach (@{$p{category}})

branches/predictive/grabbers/oztivo

@@ -3 +3 @@
 # OzTivo grabber
 
-my $version = '1.13';
+my $version = '1.14';
 
 # Requires configuration!
@@ -440 +440 @@
     my $l_min = int($local_tz / 100) * 60 + $local_tz % 100 % 60; 
     my $s_min = int($sydney_tz / 100) * 60 + $sydney_tz % 100 % 60; 
-    my $tz_diff = int(($l_min - $s_min) / 60) * 100 + (($l_min - $s_min) % 60);
+    # can't use %60 because it always returns a positive number
+    my $tz_diff = int(($l_min - $s_min) / 60) * 100 + (($l_min - $s_min) - (int(($l_min - $s_min)/60)*60));
 
     # Are we in DST?
@@ -493 +494 @@
 
     # Check Sydney's DST status each day for 7 days
-    for (my $day = 0; $day < 7; $day++)
+    for (my $day = 0; $day < 8; $day++)
     {
         my $t = $start + ($day * 24*60*60);

branches/predictive/grabbers/rex

@@ -3 +3 @@
 # "Rex"
 
-my $version  = '3.5.9';
+my $version  = '3.6.2';
 
 # An Australian TV Guide Grabber (a.k.a. tv_grab_au)
@@ -90 +90 @@
 # 3.5.7   : Bugfix : parse "rest of day" guide page correctly
 # 3.5.8   : Bugfix : don't die on failed cache write
+# 3.5.9   : Code change: use Shepherd::Common::read/write cache
+# 3.5.10  : Bugfix : don't exit on unknown option
+# 3.6.0   : Code change: use Shepherd::Common::print_stats
+#           Feature: cache guide pages for 3 hours, add subratings,
+#           Feature: capture director, writer, country, language
+#           Bugfix : fix previously-shown, aspect, subtitles, premiere, quality
+# 3.6.1   : Feature: don't split category, still add 'sport' and 'movie',
+#                    support 'LIVE' by appending to beginning of category,
+#                    use Shepherd::Common::translate_category
+# 3.6.2   : Feature: use new Common::generate_category, don't add 'advisory',
+#                    add 'length', add 'final' and 'return' (unseen)
 
 use strict;
@@ -210 +221 @@
 print stats(1);
 
+Shepherd::Common::print_stats($progname, $version, $runtime, %stats);
+
 exit 0;
 
@@ -240 +253 @@
       # Special bandwidth-saving URL for day 0
 
-      $guidedata = get_page($DATASOURCE_GUIDE_TODAY);
+      my $cacheid = $DATASOURCE_GUIDE_TODAY;
+      if ($cached->{$cacheid}) {
+        $guidedata = $cached->{$cacheid}->{data};
+      } else {
+        $guidedata = get_page($DATASOURCE_GUIDE_TODAY);
+        if ($guidedata) {
+          $cached->{$cacheid}->{data} = $guidedata;
+          $cached->{$cacheid}->{stop} = POSIX::strftime("%Y%m%d%H%M%S", localtime($runtime));
+          $cached->{$cacheid}->{title}[0][0] = $cacheid;
+        }
+      }
       parse_guide($guidedata, $date) if ($guidedata);
     }
@@ -249 +272 @@
       for (1 .. 4)
       {
-        $guidedata = get_page(url => $DATASOURCE_GUIDE,
-                        postvars => 
-                        [ 'action' => "sessionTimes",
-                          'region_id' => $rid,
-                          'date' => $date,
-                          'period' => $_,
-                          'submit' => 'submit'
-                        ]);
-        parse_guide($guidedata, $date, $_) if ($guidedata);
+        my $cacheid = "${DATASOURCE_GUIDE}?r=${rid}&d=${date}&p=$_";
+        if ($cached->{$cacheid}) {
+          $guidedata = $cached->{$cacheid}->{data};
+        } else {
+          $guidedata = get_page(url => $DATASOURCE_GUIDE,
+                        postvars =>
+                        [ 'action' => "sessionTimes",
+                          'region_id' => $rid,
+                          'date' => $date,
+                          'period' => $_,
+                          'submit' => 'submit'
+                        ]);
+          if ($guidedata) {
+            $cached->{$cacheid}->{data} = $guidedata;
+            $cached->{$cacheid}->{stop} = POSIX::strftime("%Y%m%d%H%M%S", localtime($runtime));
+            $cached->{$cacheid}->{title}[0][0] = $cacheid;
+          }
+        }
+        parse_guide($guidedata, $date, $_) if ($guidedata);
       }
     }
@@ -511 +544 @@
 sub clean_cache
 {
-    my $r = expand_date(localtime(time() - 3600)."");
+    my $r = expand_date(localtime(time() - 3*3600)."");
     my $c = 0;
     print "Removing cache items that finish earlier than $r.\n" if ($debug);
@@ -799 +832 @@
     $block = $block->right();
   }
+  my ($category, %type);
   foreach my $tag ($block->look_down('_tag' => 'tr'))
   {
@@ -804 +838 @@
     if ($str =~ /Genre:(.+)/)
     {
+      $category = strip_whitespace($1);
+      $str = $tree->find('h5');
+      $type{movie} = 1 if ($str and $str->as_text() =~ /movie/i);
+    }
+    elsif ($str =~ /Rating:(.+)/)
+    {
       $str = strip_whitespace($1);
-      if ($str eq 'N/A')
-      {
-        @extra = ( $str );
+      $str =~ /(.*?)\s*\[(.+)\]/;
+      $str = $1;
+      my $sr = $2;
+      if ($sr) {
+        $sr =~ s/or medical procedures//ig;
+        $sr = &Shepherd::Common::subrating($sr);
       }
-      else
-      {
-        @extra = split(/\//, $str);
-      }
-      $str = $tree->find('h5');
-      if ($str and $str->as_text() =~ /movie/i)
-      {
-        unshift (@extra, 'movie');
-      }
-
-      foreach (@extra)
-      {
-        $_ = [ translate_category($_), $lang ];
-      }
-      $show->{'category'} = [ @extra ];
-    }
-    elsif ($str =~ /Rating:(.+)/)
-    {
-      $str = strip_whitespace($1);
-      if ($str =~ /(.*?)\s*\[(.+)\]/)
-      {
-        @extra = split(/, /, $2);
-        foreach (@extra)
-        {
-          $_ = [ $_, 'advisory', undef ];
-        }
-        $show->{'rating'} = [[ $1, 'ABA', undef],  @extra ];
-      }
-      else
-      {
-        $show->{'rating'} = [[ $1, "ABA", undef]];
-      }
+      $show->{'rating'} = [[ $sr ? "$str $sr" : $str, 'ABA', undef]];
     }
     elsif ($str =~ /Cast: (.+)/)
@@ -846 +858 @@
       $show->{'credits'}{'actor'} = [ split(/, /, strip_whitespace($1)) ];
     }
+    elsif ($str =~ /Director: (.+)/)
+    {
+      $show->{'credits'}{'director'} = [ split(/, /, strip_whitespace($1)) ];
+    }
+    elsif ($str =~ /Writer: (.+)/) # unseen
+    {
+      $show->{'credits'}{'writer'} = [ split(/, /, strip_whitespace($1)) ];
+    }
     elsif ($str =~ /Year:\s*(\d+)/)
     {
       $show->{'date'} = strip_whitespace($1);
+    }
+    elsif ($str =~ /Country: (.+)/)
+    {
+      $show->{'country'} = [[ split(/, /, strip_whitespace($1)) ]];
+    }
+    elsif ($str =~ /Language: (.+)/)
+    {
+      $show->{'language'} = [ strip_whitespace($1) ];
+    }
+    elsif ($str =~ /Duration:\s(\d+)/)
+    {
+      $show->{'length'} = $1 * 60;
     }
     elsif ($str =~ /Other:(.+)/)
@@ -854 +886 @@
       next unless (strip_whitespace($1));
       $str = $tag->as_HTML();
-      @extra = split(/<br \/>/, $tag->as_HTML());
-      my (@to_add, %video);
+      @extra = split(/<br ?\/?>/, $tag->as_HTML());
+      my %video;
       foreach my $bit (@extra)
       {
@@ -864 +896 @@
         {
             $show->{'previously-shown'} = { };
-            next;   # Don't need this in the desc as well as it's
-                    # highlighted by MythTV
         }
         elsif ($bit eq 'Widescreen')
@@ -871 +901 @@
             $video{'aspect'} = '16:9';
         }
+        elsif ($bit eq 'High Definition')
+        {
+            $video{'quality'} = 'HDTV';
+        }
         elsif ($bit eq 'Closed Captions')
         {
             $show->{'subtitles'} = [ { 'type' => 'teletext' } ];
-            next;   # Just clutters up the desc
+        }
+        elsif ($bit eq 'New Episode')
+        {
+            $show->{'premiere'} = [ $bit ];
         }
         elsif ($bit eq 'Premiere')
         {
             $show->{'premiere'} = [ $bit ];
-        }
-        elsif ($bit eq 'High Definition')
-        {
-            $video{'quality'} = 'HDTV';
-        }
-        push (@to_add, $bit);
+            $type{premiere} = 1;
+        }
+        elsif ($bit eq 'LIVE' || $bit eq 'Live')
+        {
+            $type{live} = 1;
+        }
+        elsif ($bit eq 'Final' || $bit eq 'Finale')
+        {
+            $type{final} = 1;
+        }
+        elsif ($bit eq 'Return') # unseen
+        {
+            $type{return} = 1;
+        }
         $show->{'video'} = \%video if (%video);
       }
-      foreach (@to_add)
-      {
-        if ($show->{'desc'})
-        {
-          $show->{'desc'}[0][0] .= " $_.";
-        }
-        else
-        {
-          $show->{'desc'} = [[ "$_.", $lang ]];
-        }
-      }
-    }
-  }
+    }
+  }
+  $show->{'category'} = [ &Shepherd::Common::generate_category(
+      $show->{'title'}[0][0], $category, %type) ];
 
   $tree->delete();
@@ -919 +955 @@
   }
   return 0;
-}
-
-sub translate_category
-{
-  my %translation = (   'Sport' => 'sports',
-                        'Soap Opera' => 'Soap',
-                        'Science and Technology' => 'Science/Nature',
-                        'Real Life' => 'Reality',
-                        'Cartoon' => 'Animation',
-                        'Family' => 'Children',
-                        'Murder' => 'Crime' );
-                        
-  return $translation{$_} if $translation{$_};
-  return $_;
 }
 
@@ -1035 +1057 @@
   if (@ARGV)
   {
-    print "\nERROR: Unknown option(s): @ARGV\n";
-    help();
+    print "\nWARNING: Ignoring unknown option(s): @ARGV\n\n";
   }
 }

branches/predictive/grabbers/southerncross_website

@@ -34 +34 @@
 
 my $progname = "southerncross_website";
-my $version = "0.14";
+my $version = "0.16";
 
 #use LWP::UserAgent;
@@ -599 +599 @@
     # <b>Sunday Afternoon Premiere Movie</b> - Rat
     # <b>Sunday Afternoon Movie</b> - Evolution
-    my ($rating, $movie, $premiere, $final, $return, $series, $live);
+    my ($rating, %type);
     my $post_desc="";
     if ($subtitle and $title =~ /^(Friday|Saturday|Sunday) .* Movie$/i) {
-      $movie = 1;
-      $premiere = 1 if ($title =~ /Premiere/i);
+      $type{movie} = 1;
+      $type{premiere} = 1 if ($title =~ /Premiere/i);
       &log("parse_page debug: Changing title $title to $subtitle") 
           if $opt->{debug};
@@ -616 +616 @@
     # (Series Final)
     # (Series Return)
+    # (Replay)
     ## (New Time)
     ## (Qualifying)
+    ## (Special Presentation)
     # $subtitle =~ , live from Aurora
     if ($title =~ s/(\((.*)\))//) {
         $post_desc .= $1;
         my $tag = $2;
-        $series = 1 if ($tag =~ /(^| )Series( |$)/i);
-        $premiere = 1 if ($tag =~ /(^| )Premiere( |$)/i);
-        $final = 1 if ($tag =~ /(^| )Finale?( |$)/i);
-        $return = 1 if ($tag =~ /(^| )Return( |$)/i);
-    }
-    $live = 1 if ($subtitle and $subtitle =~ /live from/i);
+        $type{series} = 1 if ($tag =~ /(^|\W)Series(\W|$)/i);
+        $type{premiere} = 1 if ($tag =~ /(^|\W)Premiere(\W|$)/i);
+        $type{final} = 1 if ($tag =~ /(^|\W)Finale?(\W|$)/i);
+        $type{return} = 1 if ($tag =~ /(^|\W)Return(\W|$)/i);
+        $type{repeat} = 1 if ($tag =~ /(^|\W)Repeat(\W|$)/i); # unseen
+        $type{repeat} = 1 if ($tag =~ /(^|\W)Replay(\W|$)/i);
+    }
+    $type{live} = 1 if ($subtitle and $subtitle =~ /live from/i);
     $prog->{title} = [[$title, $opt->{lang}]];
     &log("parse_page debug: title     : $title $post_desc") if ($opt->{debug});
@@ -649 +653 @@
 
       if ($text) {
+        # - Live from Aurora 
         # Hosted by Mike Goldman & Bree Amer, this ... # how to parse all
         # Hosted by Mike Goldman, Ryan Fitzgerald and Bree Amer.
         # ^Guest Stars: Mel Gibson Studio ... # how to parse all
+        # Guests include: Damien Leith, Young Divas and Darren Hayes.
         # Starring: $
         ## Insert Cast: Damon Wayans, Adam Sandler, Kristen Wilson, James Caan,
         ## James Farentino (Cons Adv: Some Coarse Language, Some Violence, Sexual References)
-        # - Live from Aurora 
         ## Featuring cartoons: Yu Gi Oh! Gx (Final), All Grown Up (Repeat),
         ## Teenage Mutant Ninja Turtules (repeat)
+        $type{live} = 1 if ($text =~ /Live from/i);
         my $cast = "";
-        $cast .= "$1, " if ($text =~ /Hosted by (\w+ \w+)( |\.|,|$)/i);
-        $cast .= "$1, " if ($text =~ /Guest Stars: (\w+ \w+)( |\.|,|$)/i);
-        $cast .= $1 if ($text =~ /Starring: (.+)$/i);
-        $prog->{'credits'}{'actor'} = [ split(/, /, $cast) ] if ($cast);
-        $live = 1 if ($text =~ /Live from/i);
+        $cast .= "$1," if ($text =~ /Hosted by (\w+ \w+)( |\.|,|$)/i);
+        $cast .= "$1," if ($text =~ /Guest Stars: (\w+ \w+)( |\.|,|$)/i);
+        $cast .= "$1," if ($text =~ /Guests Include: (.+?)\./i);
+        $cast .= $1 if ($text =~ s/Starring: (.+)$//i);
+        $prog->{'credits'}{'actor'} = [ split(/,|&amp;|&|\Wand\W/i, $cast) ] if $cast;
         $prog->{desc} = [[$text, $opt->{lang}]]; # desc
         &log("parse_page debug: desc      : $text") if $opt->{debug};
@@ -682 +688 @@
         $post_desc .= $text;
         if ($rating) {
-          $rating .= " " . subrating($text);
+          $rating .= " " . &Shepherd::Common::subrating($text);
         } else {
-          $rating = subrating($text);
+          $rating = &Shepherd::Common::subrating($text);
         }
         &log("parse_page debug: verbose rating : $text") if $opt->{debug};
@@ -696 +702 @@
     }
   PART:
-    my @categories;
-    my $cft = category_from_title($title);
-    push(@categories, [$cft, $opt->{lang}]) if $cft;
-    push(@categories, ["Children", $opt->{lang}])
-        if $rating and $rating =~ /^(C|P)$/i;
-    push(@categories, ["movie", $opt->{lang}]) if $movie;
-    push(@categories, ["series", $opt->{lang}]) if $series;
-    @categories = prepend_category("Live", @categories) if $live;
-    @categories = prepend_category("Premiere", @categories) if $premiere;
-    @categories = prepend_category("Return", @categories) if $return;
-    @categories = prepend_category("Final", @categories) if $final;
-    $prog->{'category'} = [@categories] if @categories;
-    $prog->{'premiere'} = ['premiere', $opt->{lang}] if $premiere;
-    $prog->{'last-chance'} = ['final', $opt->{lang}] if $final;
-    #$prog->{'previously-shown'} = { } if $repeat; # can't detect
+    my $category = category_from_title($title);
+    $category = "Children" if !$category && $rating && $rating =~ /^(C|P)$/i;
+    $prog->{'category'} = [ &Shepherd::Common::generate_category(
+        $title, $category, %type) ];
+    $prog->{'premiere'} = ['premiere', $opt->{lang}] if $type{premiere};
+    $prog->{'last-chance'} = ['final', $opt->{lang}] if $type{final};
+    $prog->{'previously-shown'} = { } if $type{repeat};
     $prog->{rating}  = [[$rating, "ABA", undef]] if $rating;
-    if ($post_desc) {
-      if (not $prog->{desc}) {
-        $prog->{desc} = [[$post_desc, $opt->{lang}]];
-      } else {
-        $prog->{desc}->[0]->[0] .= "\n" . $post_desc;
-      }
-    }
+#     if ($post_desc) {
+#       if (not $prog->{desc}) {
+#         $prog->{desc} = [[$post_desc, $opt->{lang}]];
+#       } else {
+#         $prog->{desc}->[0]->[0] .= "\n" . $post_desc;
+#       }
+#     }
 
     $progs->{$prog->{start}} = $prog;
@@ -737 +735 @@
 {
   my $title = shift;
-  return "News" if $title=~/(^| )News( |$)/i;
-  return "sports" if $title=~/(^| )Sports Tonight( |$)/i;
-  return "sports" if $title=~/(^| )AFL( |$)/i;
-  return "Infotainment" if $title=~/(^| )Infomercials?( |$)/i;
-  return "Reality" if $title=~/(^| )Big Brother( |$)/i;
-  return "Animation" if $title=~/(^| )The( All New)? Simpsons( |$)/i;
-  return "Music" if $title=~/(^| )Video Hits( |$)/i;
-}
-
-sub prepend_category
-{
-  my ($item, @categories) = @_;
-
-  if (@categories > 0 and $categories[0][0]) {
-    if ($categories[0][0] eq "movie") { unshift(@categories, ["Movie", $opt->{lang}]) }
-    elsif ($categories[0][0] eq "sports") { unshift(@categories, ["Sports", $opt->{lang}]) }
-    elsif ($categories[0][0] eq "series") { unshift(@categories, ["Series", $opt->{lang}]) }
-    elsif ($categories[0][0] eq "tvshow") { unshift(@categories, ["TVShow", $opt->{lang}]) }
-    $categories[0][0] = $item . " " . $categories[0][0];
-  } else {
-    push(@categories, [$item, $opt->{lang}]);
-  }
-  return @categories;
-}
-
-# (Adult Themes)
-# (Some Violence, Adult Themes, Supernatural Themes)
-# (Drug References, Adult Themes)
-# (Very Coarse Language, Sexual References, Drug References, Adult Themes, Nudity)
-# (Some Violence)
-# (Drug Use, Strong Adult Themes)
-# (Some Violence, Adult Themes)
-# (Some Coarse Language)
-# (Sexual References)
-# (Mild Coarse Language, Sexual References)
-# (Sex Scenes, Adult Themes, Supernatural Themes)
-## (Qualifying - Sat)
-sub subrating
-{
-  my $string = shift;
-  my @subrating;
-
-  push(@subrating, "v") if $string =~ /Violence/i;
-  push(@subrating, "l") if $string =~ /Language/i;
-  push(@subrating, "s") if $string =~ /Sex/i;
-  push(@subrating, "d") if $string =~ /Drug/i;
-  push(@subrating, "a") if $string =~ /Adult/i;
-  push(@subrating, "n") if $string =~ /Nudity/i;
-  push(@subrating, "h") if $string =~ /Supernatural/i;
-
-  return join(",",@subrating);
-}
+  return "News" if $title=~/(^|\W)News(\W|$)/i;
+  return "sports" if $title=~/(^|\W)Sports Tonight(\W|$)/i;
+  return "sports" if $title=~/(^|\W)AFL(\W|$)/i;
+  return "Infotainment" if $title=~/(^|\W)Infomercials?(\W|$)/i;
+  return "Reality" if $title=~/(^|\W)Big Brother(\W|$)/i;
+  return "Animation" if $title=~/(^|\W)The( All New)? Simpsons(\W|$)/i;
+  return "Music" if $title=~/(^|\W)Video Hits(\W|$)/i;
+  return "Soap" if $title=~/(^|\W)The Bold (&amp;|&|and) The Beautiful(\W|$)/i;
+  return "Lifestyle/Cooking, Food & Wine" if $title=~/(^|\W)Hueys Cooking Adventures(\W|$)/i;
+  return "Variety" if $title=~/(^|\W)The Late Show With David Letterman(\W|$)/i;
+}

branches/predictive/grabbers/yahoo7widget

@@ -22 +22 @@
 
 my $progname = "yahoo7widget";
-my $version = "1.92";
+my $version = "1.93";
 
 use XMLTV;
@@ -512 +512 @@
                         foreach my $field ('title', 'subtitle', 'description_1', 'main_cast', 'year_released', 'rating',
                           'genre', 'running_time', 'repeat', 'country', 'movie', 'premiere', 'final', 'captions', 'warnings', 
-                          'color', 'language', 'director', 'live') {
+                          'color', 'language', 'director', 'live', 'return') {
                                 eval { $e{$field} = $event->getElementsByTagName("$field")->item(0)->getFirstChild->getNodeValue; };
                         }
                         # other fields we dont pick up but exist in source xml data include:
                         #  other_title, description_2, genre_id, highlight, ext_url, y7_url
-
-                        my @categories;
-
-                        push(@categories, [ "movie",$opt->{lang} ] ) if (($e{movie}) && ($e{movie} == 1));
-                        push(@categories, [ "final",$opt->{lang} ] ) if (($e{final}) && ($e{final} == 1));
-                        push(@categories, [ "live",$opt->{lang} ] ) if (($e{live}) && ($e{live} == 1));
-                        push(@categories, [ Shepherd::Common::translate_category($e{genre}),$opt->{lang} ] ) if (($e{genre}) && ($e{genre} ne ""));
+                        # 'return' unseen
 
                         my %video_details;
@@ -542 +536 @@
                         $tv_guide->{$channel}->{data}->{$event_id}->{'sub-title'} =     [[ $e{subtitle}, $opt->{lang} ]] if $e{subtitle};
                         $tv_guide->{$channel}->{data}->{$event_id}->{'desc'} =          [[ $e{description_1}, $opt->{lang} ]] if $e{description_1};
-                        $tv_guide->{$channel}->{data}->{$event_id}->{'category'} =      [ @categories ] if @categories;
+                        $tv_guide->{$channel}->{data}->{$event_id}->{'category'} =      [ &Shepherd::Common::generate_category($e{title}, $e{genre}, %e) ];
                         $tv_guide->{$channel}->{data}->{$event_id}->{'country'} =       [[ $e{country}, $opt->{lang} ]] if $e{country};
                         $tv_guide->{$channel}->{data}->{$event_id}->{'premiere'} =      [ 'premiere', $opt->{lang} ] if $e{premiere};
@@ -548 +542 @@
                         $tv_guide->{$channel}->{data}->{$event_id}->{'credits'}{'actor'} = [ split(/, /, $e{main_cast}) ] if $e{main_cast};
                         $tv_guide->{$channel}->{data}->{$event_id}->{'credits'}{'director'} = [ split(/, /, $e{director}) ] if $e{director};
+                        $tv_guide->{$channel}->{data}->{$event_id}->{'credits'}{'writer'} = [ split(/, /, $e{writer}) ] if $e{writer}; # unseen
                         $tv_guide->{$channel}->{data}->{$event_id}->{'date'} =  $e{year_released} if $e{year_released};
                         $tv_guide->{$channel}->{data}->{$event_id}->{'previously-shown'} = { } if $e{repeat};

branches/predictive/reconcilers/reconciler_mk2

@@ -96 +96 @@
 
 my $progname = "reconciler_mk2";
-my $version = "0.35";
+my $version = "0.36";
 
 use LWP::UserAgent;
@@ -172 +172 @@
 $reclogic{warn_on_overlapping_programmes_for_different_title} = 1; # do warn on overlapping programmes from same source with different names
 $reclogic{warn_on_invalid_time_in_prog} = 1;            # do warn on a programme with a duration that is too long
+$reclogic{warn_on_tba_prog} = 1;                        # do warn on encountering To Be Advised shows
 
 #
@@ -1061 +1062 @@
                 $stats{bad_programme_duration_too_long}++;
                 return;
+        }
+
+        # skip if 'To Be Advised' or 'TBA'
+        if ($prog_title =~ /^to be advised$/i or $prog_title =~ /^tba$/i)
+        {
+            &log($reclogic{warn_on_tba_prog},
+                 (sprintf "WARNING: ignoring TBA programme '%s' on channel '%s' from %s: start '%s' stop '%s'; ignored.",
+                 $prog_title, $prog_chan, $datafile, 
+                 ($prog->{start} ? $prog->{start} : "undef"), 
+                 ($prog->{stop} ? $prog->{stop} : "undef")));
+            return;
         }
 

branches/predictive/references/Shepherd/Common.pm

@@ -3 +3 @@
 # Shepherd::Common library
 
-my $version = '0.27';
+my $version = '0.29';
 
 #
@@ -770 +770 @@
 ##########################################################################
 
+# if no category then guess from title for Sport, News, Infomercial
+# translates the words in category
+# types (final,premiere,return,live) are prepend to category
+# types (movie,sports,series,tvshow) are appended to category list
+sub generate_category
+{
+    my ($title, $category, %type) = @_;
+
+    $type{sports} = 1 if ($title && $title=~/(^|\W)Sports?(\W|$)/i);
+    $type{sports} = 1 if ($category && $category=~/(^|\W)Sports?(\W|$)/i);
+
+    if ($category) {
+        if ($category eq "movie") {
+            $category = "Movie";
+            $type{movie} = 1;
+        } elsif ($category eq "sports") {
+            $category = "Sports";
+            $type{sports} = 1;
+        } elsif ($category eq "series") {
+            $category = "Series";
+            $type{series} = 1;
+        } elsif ($category eq "tvshow") {
+            $category = "TVShow";
+            $type{tvshow} = 1;
+        }
+        $category =~ s/Soap Opera/Soap/ig;
+        $category =~ s/Science and Technology/Science\/Nature/ig;
+        $category =~ s/Real Life/Reality/ig;
+        $category =~ s/Cartoon/Animation/ig;
+        $category =~ s/Family/Children/ig;
+        $category =~ s/Murder/Crime/ig;
+    } else { # !$category
+        if ($title) {
+            if ($title=~/(^|\W)News(\W|$)/i) {
+                $category = "News";
+            } elsif ($title=~/(^|\W)Infomercials?(\W|$)/i) {
+                $category = "Infotainment";
+            }
+        }
+        if (!$category) {
+            if ($type{movie}) {
+                $category = "Movie";
+            } elsif ($type{sports}) {
+                $category = "Sports";
+            } elsif ($type{series}) {
+                $category = "Series";
+            } elsif ($type{tvshow}) {
+               $category = "TVShow";
+            }
+        }
+    }
+
+    $category = "" if (!$category);
+    $category = "Live $category" if ($type{live});
+    $category = "Return $category" if ($type{return});
+    $category = "Premiere $category" if ($type{premiere});
+    $category = "Final $category" if ($type{final});
+    $category =~ s/^\s*(.*?)\s*$/$1/;
+
+    my @result;
+    @result = [ $category, "en"] if $category;
+    push(@result, [ "movie"  ]) if $type{movie};
+    push(@result, [ "sports" ]) if $type{sports};
+    push(@result, [ "series" ]) if $type{series};
+    push(@result, [ "tvshow" ]) if $type{tvshow};
+
+    return @result;
+}
+
+##########################################################################
+
+# (Adult Themes)
+# (Some Violence, Adult Themes, Supernatural Themes)
+# (Drug References, Adult Themes)
+# (Very Coarse Language, Sexual References, Drug References, Adult Themes, Nudity)
+# (Some Violence)
+# (Drug Use, Strong Adult Themes)
+# (Some Violence, Adult Themes)
+# (Some Coarse Language)
+# (Sexual References)
+# (Mild Coarse Language, Sexual References)
+# (Sex Scenes, Adult Themes, Supernatural Themes)
+# (Adult Themes, Medical Procedures)
+## (Qualifying - Sat)
+sub subrating
+{
+  my $string = shift || "";
+
+  my @subrating;
+  push(@subrating, "v") if $string =~ /Violence/i;
+  push(@subrating, "l") if $string =~ /Language/i;
+  push(@subrating, "s") if $string =~ /Sex/i;
+  push(@subrating, "d") if $string =~ /Drug/i;
+  push(@subrating, "a") if $string =~ /Adult/i;
+  push(@subrating, "n") if $string =~ /Nudity/i;
+  push(@subrating, "h") if $string =~ /Horror|Supernatural/i;
+  push(@subrating, "m") if $string =~ /Medical/i;
+
+  return join(",",@subrating);
+}
+
+##########################################################################
+
 sub log
 {

branches/predictive/status

@@ -1 @@
-application     shepherd            1.2.24
+application     shepherd            1.2.26
 reference       channel_list        6
-reference       Shepherd/Common.pm  0.27
+reference       Shepherd/Common.pm  0.29
 reference       Shepherd/MythTV.pm  0.6
-grabber         yahoo7widget        1.92
-grabber         rex                 3.5.9
-grabber         abc_website         3.14
+grabber         yahoo7widget        1.93
+grabber         rex                 3.6.2
+grabber         abc_website         3.14-r1
 grabber         sbsnews_website     0.18
-grabber         oztivo              1.13
+grabber         oztivo              1.14
 grabber         jrobbo              0.16
 grabber         yahoo7web           0.28-r1
 grabber         foxtel_swf          0.24
 grabber         channelnsw_gov      0.8
-grabber         southerncross_website   0.14
+grabber         southerncross_website   0.16
 grabber         ten_website         1.04
 grabber         selectv_website     0.02
-grabber         citysearch          2.0.2
-grabber         acctv_website       0.02
-reconciler      reconciler_mk2      0.35
+grabber         citysearch          2.0.3
+grabber         acctv_website       0.03
+reconciler      reconciler_mk2      0.36
 postprocessor   imdb_augment_data   0.35
 postprocessor   tvdb_augment_data   0.11