Changeset 74

Timestamp:

10/16/06 17:20:07 (7 years ago)

Author:

lincoln

Message:

reconciler_mk2 enhancement: compress logfiles automatically since they are quite large

Files:

• reconcilers/reconciler_mk2 (modified) (18 diffs)
• status (modified) (1 diff)

reconcilers/reconciler_mk2

@@ -16 +16 @@
 #    0.04  17aug06      bug fix: use our own parse_xmltv_date, Date::Manip seems broken
 #                       bug fix: correctly parse some XMLTV rare fields 
+#    0.05  17aug06      enhancement: compress logfiles automatically since they are quite large
 
 #
@@ -75 +76 @@
 
 my $progname = "reconciler_mk2";
-my $version = "0.04_15aug06";
+my $version = "0.05_17aug06";
 
 use LWP::UserAgent;
@@ -197 +198 @@
         'lang=s'                => \$opt->{lang},
         'option=s'              => \%reclogic,
+
+        'no-log'                => \$opt->{nolog},
 
         'print_listing'         => \$reclogic{debug_print_programme_list},
@@ -256 +259 @@
 
 # go go go!
-open(LOG_FILE,">$opt->{log_file}") || die "can't open log file $opt->{log_file} for writing: $!\n";
-open(ALT_TITLE_FILE,">$opt->{alt_title_file}") || die "can't open alt-title-file $opt->{alt_title_file} for writing: $!\n";
+unless ($opt->{nolog}) {
+        &rotate_logfiles;
+
+        open(LOG_FILE,">$opt->{log_file}") || die "can't open log file $opt->{log_file} for writing: $!\n";
+        open(ALT_TITLE_FILE,">$opt->{alt_title_file}") || die "can't open alt-title-file $opt->{alt_title_file} for writing: $!\n";
+
+        printf ALT_TITLE_FILE "$progname $version\n";
+        printf ALT_TITLE_FILE "This log file contains lists of what title matches were attempted (but didn't match).\n";
+        printf ALT_TITLE_FILE "The intent of this is to keep a log that we can (hopefully) interpret in order to\n";
+        printf ALT_TITLE_FILE "improve the alternateTitle() logic.\n\n";
+}
 
 &log(1,(sprintf "started: %s%soutput %s",
@@ -263 +275 @@
         ($opt->{config_file} ? "default overrides in $opt->{config_file}, " : ""),
         ($opt->{output_file})));
-&log(1,(sprintf "  logging to %s, alternate titles to %s",$opt->{log_file},$opt->{alt_title_file}));
-
-printf ALT_TITLE_FILE "$progname $version\n";
-printf ALT_TITLE_FILE "This log file contains lists of what title matches were attempted (but didn't match).\n";
-printf ALT_TITLE_FILE "The intent of this is to keep a log that we can (hopefully) interpret in order to\n";
-printf ALT_TITLE_FILE "improve the alternateTitle() logic.\n\n";
+&log(1,(sprintf "  logging to %s, alternate titles to %s",$opt->{log_file},$opt->{alt_title_file})) unless $opt->{nolog};
 
 &override_settings;
@@ -293 +300 @@
 &print_stats;
 
-close(LOG_FILE);
-close(ALT_TITLE_FILE);
+unless ($opt->{nolog}) {
+        close(LOG_FILE);
+        close(ALT_TITLE_FILE);
+
+        compress_file($opt->{log_file});
+        compress_file($opt->{alt_title_file});
+}
 
 exit(0);
@@ -360 +372 @@
 
 ######################################################################################################
+
+sub rotate_logfiles
+{
+        # keep last 4 log files
+        foreach my $file ($opt->{log_file}, $opt->{alt_title_file}) {
+                my $num;
+                for ($num = 4; $num > 0; $num--) {
+                        my $f1 = sprintf "%s.%d.gz",$file,$num;
+                        my $f2 = sprintf "%s.%d.gz",$file,$num+1;
+                        unlink($f2);
+                        rename($f1,$f2);
+                }
+
+                my $f1 = sprintf "%s.gz",$file;
+                my $f2 = sprintf "%s.1.gz",$file;
+                rename($f1,$f2);
+        }
+}
+
+######################################################################################################
+
+sub compress_file
+{
+        my $infile = shift;
+        my $outfile = sprintf "%s.gz",$infile;
+        my $gz;
+
+        if (!(open(INFILE,"<$infile"))) {
+                warn "could not open file $infile for reading: $!\n";
+                return;
+        }
+
+        if (!($gz = gzopen($outfile,"wb"))) {
+                warn "could not open file $outfile for writing: $!\n";
+                return;
+        }
+
+        while (<INFILE>) {
+                my $byteswritten = $gz->gzwrite($_);
+                if ($byteswritten == 0) {
+                        warn "error writing to compressed file: error $gz->gzerror";
+                }
+        }
+        close(INFILE);
+        $gz->gzclose();
+        unlink($infile);
+}
+
+######################################################################################################
 # if we are supplied a channels_file, then fill in channels we know about from that.
 # this is useful because some grabbers (e.g. oztivo) don't supply a <channels></channels>
@@ -386 +447 @@
         my ($log_to_screen,$entry) = @_;
         printf STDERR "%s [%d] %s\n",$progname, time,$entry if ($log_to_screen);
-        printf LOG_FILE "%s [%d] %s\n",$progname, time,$entry;
+        printf LOG_FILE "%s [%d] %s\n",$progname, time,$entry unless $opt->{nolog};
 }
 
@@ -760 +821 @@
         if (($t2 - $t1) > $reclogic{max_programme_length}) {
                 &log($reclogic{warn_on_invalid_time_in_prog},
-                  (sprintf "WARNING: programme '%s' on channel '%s' from %s had a programme duration (%d) that exceeded max_programme_length (%d): start '%s' stop '%s'; ignored.\n",
+                  (sprintf "WARNING: programme '%s' on channel '%s' from %s had a programme duration (%d) that exceeded max_programme_length (%d): start '%s' stop '%s'; ignored.",
                   $prog_title, $prog_chan, $datafile, ($t2-$t1),  $reclogic{max_programme_length},
                   ($prog->{start} ? $prog->{start} : "undef"), ($prog->{stop} ? $prog->{stop} : "undef")));
@@ -806 +867 @@
                 &log($warn,(sprintf "WARNING: file contained more than one programme in same timeslot: channel '%s' start %s, stop %s, programs \"%s\" and \"%s\"; ignored.",
                         $prog_chan,
-                        (strftime "%a %e %b %H:%M",localtime($t1)),
-                        (strftime "%H:%M",localtime($t2)),
+                        (strftime "%a%e%b%H:%M",localtime($t1)),
+                        (strftime "%a%e%b%H:%M",localtime($t2)),
                         $prog_title,
                         $in->{dupcheck}->{$source}->{$prog_chan}->{$prog_key}));
@@ -878 +939 @@
                         printf "%-10s %s - %s: %-30s {src %s}\n",
                                 $in->{channels}->{$chan_id}->{pref_desc},
-                                (strftime "%a%e%b %H:%M", localtime($prog->{start_epoch})),
-                                (strftime "%H:%M", localtime($prog->{stop_epoch})),
+                                (strftime "%a%e%b%H:%M", localtime($prog->{start_epoch})),
+                                (strftime "%a%e%b%H:%M", localtime($prog->{stop_epoch})),
                                 $progname,
                                 $prog->{datasources}
@@ -903 +964 @@
         &log($reclogic{debug_add_logic},
           (sprintf "add_one_programme: adding programme %s-%s \"%s\" with data from %d programmes",
-          (strftime "%a %e %b %H:%M",localtime(($m->[0]->{start_epoch}))),
-          (strftime "%H:%M",localtime(($m->[0]->{stop_epoch}))),
+          (strftime "%a%e%b%H:%M",localtime(($m->[0]->{start_epoch}))),
+          (strftime "%a%e%b%H:%M",localtime(($m->[0]->{stop_epoch}))),
           $m->[0]->{title}->[0]->[0], $num_matching));
 
@@ -1114 +1175 @@
                                 &log($reclogic{debug_delete_logic},
                                   (sprintf " - deleting \"%s\" (%s-%s, grabber %d) - within window",
-                                  $prog_title, (strftime "%a%e%b %H:%M", localtime($prog_start)),
-                                  (strftime "%H:%M", localtime($prog_stop)), $prog_grabber));
+                                  $prog_title, (strftime "%a%e%b%H:%M", localtime($prog_start)),
+                                  (strftime "%a%e%b%H:%M", localtime($prog_stop)), $prog_grabber));
                         } elsif (($prog_start < $del_start) && ($prog_stop > $del_stop)) {
                                 #
@@ -1132 +1193 @@
                                         &log($reclogic{debug_delete_logic},
                                           (sprintf " - deleting \"%s\" (%s-%s, grabber %d) - within window +/- delete_window_smaller_than_prog_window_threshold (%d)",
-                                          $prog_title, (strftime "%a%e%b %H:%M", localtime($prog_start)),
-                                          (strftime "%H:%M", localtime($prog_stop)), $prog_grabber,
+                                          $prog_title, (strftime "%a%e%b%H:%M", localtime($prog_start)),
+                                          (strftime "%a%e%b%H:%M", localtime($prog_stop)), $prog_grabber,
                                           $reclogic{delete_window_smaller_than_prog_window_threshold}));
                                         $stats{prog_del_overwindow_close_enough}++;
@@ -1322 +1383 @@
                         my $alternate_titles = sprintf "channel %s timeslot %s-%s, preferred grabber %d\n",
                           $in->{channels}->{$chan_id}->{pref_desc},
-                          (strftime "%a %e %b %H:%M",localtime(($prog->{start_epoch}))),
-                          (strftime "%H:%M",localtime(($prog->{stop_epoch}))),
+                          (strftime "%a%e%b%H:%M",localtime(($prog->{start_epoch}))),
+                          (strftime "%a%e%b%H:%M",localtime(($prog->{stop_epoch}))),
                           $prog->{grabber_num};
 
@@ -1400 +1461 @@
                                 &log($reclogic{debug_reconcile_pass1},
                                   (sprintf "reconcile:   %d programs between %s - %s",
-                                  $num_progs_in_slot, (strftime "%a %e %b %H:%M",localtime($this_prog_start)),
-                                  (strftime "%H:%M",localtime($this_prog_stop))));
+                                  $num_progs_in_slot, (strftime "%a%e%b%H:%M",localtime($this_prog_start)),
+                                  (strftime "%a%e%b%H:%M",localtime($this_prog_stop))));
 
                                 $stats{rec_start_stop_match}++;
@@ -1439 +1500 @@
                                   (sprintf "reconcile: (%d to go) chose programme for %s-%s: \"%s\", looking for alternatives between %s-%s",
                                   $in->{total_progs}->{$chan_id},
-                                  (strftime "%a %e %b %H:%M",localtime($found_prog->{start_epoch})),
-                                  (strftime "%H:%M",localtime($found_prog->{stop_epoch})), $found_prog->{title}->[0]->[0],
-                                  (strftime "%H:%M",localtime($start_window)),
-                                  (strftime "%H:%M",localtime($stop_window))));
+                                  (strftime "%a%e%b%H:%M",localtime($found_prog->{start_epoch})),
+                                  (strftime "%a%e%b%H:%M",localtime($found_prog->{stop_epoch})), $found_prog->{title}->[0]->[0],
+                                  (strftime "%a%e%b%H:%M",localtime($start_window)),
+                                  (strftime "%a%e%b%H:%M",localtime($stop_window))));
 
                                 foreach my $prog_key2 ( sort { $in->{tree}->{$chan_id}->{$a} <=> $in->{tree}->{$chan_id}->{$b} } keys %{($in->{tree}->{$chan_id})} ) {
@@ -1459 +1520 @@
                                                         &log($reclogic{debug_reconcile_pass2},
                                                           (sprintf "reconcile:     found alternative prog: %s-%s, \"%s\"",
-                                                          (strftime "%a%e%b %H:%M",localtime($check_prog->{start_epoch})),
-                                                          (strftime "%H:%M",localtime($check_prog->{stop_epoch})),
+                                                          (strftime "%a%e%b%H:%M",localtime($check_prog->{start_epoch})),
+                                                          (strftime "%a%e%b%H:%M",localtime($check_prog->{stop_epoch})),
                                                           $check_prog->{title}->[0]->[0]));
                                                 }

status

@@ -7 +7 @@
 jrobbo:0.03-r2:grabber
 d1:0.6.2.3-r2:grabber
-reconciler_mk2:0.04:reconciler
+reconciler_mk2:0.05:reconciler
 imdb_augment_data:0.01:postprocessor