Changeset 34 for shepherd

Timestamp:

10/06/06 19:09:57 (7 years ago)

Author:

lincoln

Message:

a few fixes/enhancements:
(1) new command line option --dontcallgrabbers - useful for when debugging/testing,

means you can test using previously-collected xmltv output from each grabber chosen

(2) data-scrubber automatically fixes oztivo's bogus (blank) 'director' lines

to stop XMLTV whining

(3) data-scrubber removes start_epoch/stop_epoch before write_programme to stop XMLTV whining

Files:

• shepherd (modified) (7 diffs)

shepherd

@@ -271 +271 @@
         $comm .= " --debug" if ($debug);
         $comm .= " @ARGV" if (@ARGV);
-        print "SHEPHERD: Excuting command: $comm\n";
-
-        chdir "$GRABBER_DIR/$grabber/";
-        system($comm);
-        chdir $CWD;
+
+        if ((defined $opt->{dontcallgrabbers}) && ($opt->{dontcallgrabbers})) {
+            printf "SHEPHERD: not calling grabber because of --dontcallgrabbers option, but will instead use existing $output\n";
+        } else {
+            print "SHEPHERD: Excuting command: $comm\n";
+            chdir "$GRABBER_DIR/$grabber/";
+            system($comm);
+            chdir $CWD;
+        }
 
         # soak up the data we just collected
@@ -737 +741 @@
                             next if ($field eq "stop_epoch");
                             if (!defined $new_prog_entry->{$field}) {
-                                printf "REC#5b:      adding field \"%s\"\n",$field;
+                                printf "REC#5b:      adding field \"%s\"\n",$field if $recdebug;
                                 $new_prog_entry->{$field} = $match_prog->{$field};
                                 # TODO (FUTURE): should we add to programme description to say where we got what data from?
@@ -745 +749 @@
                 }
 
+                #
                 # 6.  write out new entry
+                #
+
                 printf "REC#6: writing out programme entry\n" if $recdebug;
-                #delete $new_prog_entry->{'start_epoch'};
-                #delete $new_prog_entry->{'stop_epoch'};
                 &cleanup($new_prog_entry);
+
+                # scrub programme for known bogosities
+
+                # oztivo typically inserts blank 'director' details into 'credits' .. scrub them
+                if ((defined $new_prog_entry->{'credits'}) &&
+                    (defined $new_prog_entry->{'credits'}->{'director'}) &&
+                    (defined $new_prog_entry->{'credits'}->{'director'}->[0])) {
+                    my @director_list = $new_prog_entry->{'credits'}->{'director'}->[0];
+                    for my $i (0 .. $#director_list) {
+                        delete $new_prog_entry->{'credits'}->{'director'}->[$i] if ((defined $director_list[$i]) && ($director_list[$i] eq ""));
+                    }
+                }
+
+                # want to keep epoch start/stop for our own processing, but stop XMLTV whining about it in write_programme
+                # so temporarily remove them & reinsert them back afterwards
+                my ($orig_start_epoch,$orig_end_epoch) = ($new_prog_entry->{'start_epoch'},$new_prog_entry->{'stop_epoch'});
+                delete $new_prog_entry->{'start_epoch'};
+                delete $new_prog_entry->{'stop_epoch'};
+
+                # write out
                 $writer->write_programme($new_prog_entry);
+                ($new_prog_entry->{'start_epoch'},$new_prog_entry->{'stop_epoch'}) = ($orig_start_epoch,$orig_end_epoch);
 
                 # 7a. remove all programmes that end before this endtime
@@ -823 +849 @@
 my %amp;
 BEGIN { %amp = ( nbsp => ' ', qw{ amp & lt < gt > apos ' quot " } ) }
+
 sub cleanup {
     my $x = shift;
@@ -829 +856 @@
     elsif (ref $x eq "ARRAY") { cleanup(\$_) for @$x }
     elsif (defined $$x) {
-        $$x =~ s/&(#(\d+)|(.*?));/ $2 ? chr($2) : $amp{$3}||' ' /eg;
-        # $$x =~ s/[^\x20-\x7f]/ /g;
-        $$x =~ s/(^\s+|\s+$)//g;
-    }
-}
-
+        $$x =~ s/&(#(\d+)|(.*?));/ $2 ? chr($2) : $amp{$3}||' ' /eg; # scrub html
+        # $$x =~ s/[^\x20-\x7f]/ /g;    # disabled (we want to keep non-std chars)
+        $$x =~ s/(^\s+|\s+$)//g;        # strip leading/trailing spaces
+    }
+}
 
 # -----------------------------------------
@@ -1339 +1365 @@
               'configure'       => \$opt->{configure},
               'mirror=s'        => \$opt->{mirror},
+              'dontcallgrabbers' => \$opt->{dontcallgrabbers},
               'debug'           => \$debug);
 }
@@ -1581 +1608 @@
     } else {
         if (!ref($arg)) {
-            CORE::die((sprintf "DIE at line %d in file %s: %s\n",$line,$file,(join("",($arg,@rest)))));
+            printf STDERR "DIE at line %d in file %s\n",$line,$file;
+            CORE::die(join("",@rest));
         } else {
             CORE::die($arg,@rest);