Changeset 710

Timestamp:

05/29/07 07:05:37 (6 years ago)

Author:

lincoln

Message:

ten_website is back

Files:

• grabbers/ten_website (modified) (12 diffs)
• status (modified) (1 diff)

grabbers/ten_website

@@ -107 +107 @@
 
 # set defaults
-Shepherd::Common::set_default("debug", $opt->{debug}) if (defined $opt->{debug});
+Shepherd::Common::set_default("debug", (defined $opt->{debug} ? 2 : 0));
 Shepherd::Common::set_default("webwarper", 1) if (defined $opt->{warper});
 Shepherd::Common::set_default("squid", 1) if (defined $opt->{obfuscate});
@@ -166 +166 @@
         --cache-file=file       where to store cache (default "$opt->{cache_file}")
         --fast                  don't run slow - get data as quick as you can - not recommended
-        --anonsocks=(ip:port)   use SOCKS4A server at (ip):(port) (for Tor: recommended)
 
         --debug                 increase debug level
@@ -258 +257 @@
               'generator-info-name' => "$progname $version"} );
 
-        $writer->write_channel( {
-                'display-name' => [[ "TEN", $opt->{lang} ]], 'id' => $channels->{TEN} } );
+        $writer->write_channel( {'display-name' => [[ "TEN", $opt->{lang} ]], 'id' => $channels->{TEN} } ) if (defined $channels->{TEN});
+        $writer->write_channel( {'display-name' => [[ "TEN", $opt->{lang} ]], 'id' => $opt_channels->{TEN} } ) if (defined $opt_channels->{TEN});
 
         foreach my $prog (@{($d->{progs})}) {
@@ -282 +281 @@
         $reg = "adelaide" if ($opt->{region} =~ /(81|82|83|85|86|107)/);        # adelaide
 
-        &log("fetching summary page");
+        &log("fetching summary page (".$reg.")");
+        $stats{programmes} = 0;
 
         my $url = "http://ten.com.au/tv-schedule/full?location=".$reg."&uid=";
@@ -323 +323 @@
                 my $prog_start = substr($p->{startTime},0,10);
                 my $prog_stop =  substr($p->{endTime},0,10);
-                if (($prog_start < $starttime) || ($prog_start > $endtime)) {
+
+                # only fetch within start/end times specified.
+                if (($prog_stop < $starttime) || ($prog_start > $endtime)) {
                         $stats{prog_outside_window}++;
                         next;
+                }
+
+                # if microgap fetching only fetch within gaps
+                if (defined $opt->{gaps_file}) {
+                        my $found_gap = 0;
+                        foreach my $g (@{($gaps->{TEN})}) {
+                                my ($s, $e) = split(/-/,$g);
+                                if ((($s >= $prog_start) && ($s <= $prog_stop)) ||
+                                    (($e >= $prog_start) && ($e <= $prog_stop)) ||
+                                    (($s <= $prog_start) && ($e >= $prog_stop))) {
+                                        $found_gap = 1;
+                                }
+                        }
+                        if (!$found_gap) {
+                                $stats{gaps_skipped}++;
+                                next;
+                        }
                 }
 
@@ -342 +361 @@
                         if (($p->{link} =~ /id=Hillsong/) ||
                             ($p->{link} =~ /id=This_Is_Your_Day_With_Benny_Hinn/) ||
+                            ($p->{link} =~ /id=Kenneth_Copeland/) ||
+                            ($p->{link} =~ /id=Life_Today_With_James_Robison/) ||
                             ($p->{link} =~ /id=Christian_City_TV/)) {
                                 push(@categories, [ "Religion", $opt->{lang} ] );
                         } elsif ($p->{link} =~ /id=Home_Shopping/) {
                                 push(@categories, [ "Shopping", $opt->{lang} ] );
+                        } elsif ($p->{link} =~ /id=Toasted_TV/) {
+                                push(@categories, [ "Kids", $opt->{lang} ] );
                         }
                 }
@@ -351 +374 @@
                 # "schedule_id", "series_id"
 
-                push(@{($d->{progs})}, $prog);
+                $d->{progs}->[$stats{programmes}] = $prog;
                 $stats{programmes}++;
         }
 
-        print " - fetched ".$stats{programmes}." programs: ".Dumper($d) if (defined $opt->{debug});
+        &log(" summary returned data for ".$stats{programmes}." programmes");
 }
 
@@ -366 +389 @@
         &log("fetching up to ".$stats{programmes}." detail pages...");
 
-        foreach my $prog (@{($d->{progs})}) {
-                my $url = $prog->{link};
+        for (my $i=0; $i < $stats{programmes}; $i++) {
+                my $url = $d->{progs}->[$i]->{link};
                 my $was_in_cache = 0;
                 $prog_count++;
                 next if ((!defined $url) || ($url eq ""));
 
+                # some descriptions all end up being the same.  just skip fetching details on these
                 if (($url =~ /id=Home_Shopping/) ||
                     ($url =~ /id=Hillsong/) ||
                     ($url =~ /id=This_Is_Your_Day_With_Benny_Hinn/) ||
-                    ($url =~ /id=Christian_City_TV/)) {
+                    ($url =~ /id=Kenneth_Copeland/) ||
+                    ($url =~ /id=Life_Today_With_James_Robison/) ||
+                    ($url =~ /id=Christian_City_TV/) ||
+                    ($url =~ /id=Toasted_TV/)) {
                         $stats{skipped_detail_pages}++;
                         next;
@@ -385 +412 @@
 
                 if (!defined $data_cache->{prog_cache}->{$url}) {
-                        my $url = "http://ten.com.au".$url;
+                        my $fetch_url = "http://ten.com.au".$url;
                         my $tries = 3;
 
-                        &log("fetching prog ".$prog_count." [".$url."] ..") if (defined $opt->{debug} && $opt->{debug} > 1);
+                        &log("fetching prog ".$prog_count." [".$fetch_url."] ..") if (defined $opt->{debug} && $opt->{debug} > 1);
 
                         my ($data, $success, $status_msg, $bytes_fetched, $seconds_slept, $failed_attempts, $mime_type) =
-                          Shepherd::Common::get_url(url => $url, retries => ($tries-1));
+                          Shepherd::Common::get_url(url => $fetch_url, retries => ($tries-1));
 
                         $stats{failed_requests} += $failed_attempts;
@@ -398 +425 @@
 
                         if ((!$data) || (!$success)) {
-                                &log("Failed to fetch '$url' after $tries attempts. Has the format changed?");
+                                &log("Failed to fetch '$fetch_url' after $tries attempts. Has the format changed?");
                                 $stats{bad_detail_response}++;
+
+                                if ($stats{bad_detail_response} >= 3) {
+                                        &log($stats{bad_detail_response}." bad detailed responses.  Disabling fetching details.");
+                                        return;
+                                }
+
                                 next;
                         }
@@ -411 +444 @@
                         my $prog_desc = $tree->look_down('_tag' => 'td', 'class' => 'bottom-row');
                         if (defined $prog_desc) {
-                                $data_cache->{prog_cache}->{$url}->{desc} = [[ $prog_desc->as_text(), $opt->{lang} ]];
-                                &log("  got desc '".$prog_desc->as_text()."'") if (defined $opt->{debug} && $opt->{debug} > 1);
+                                my $prog_desc_text = $prog_desc->as_text();
+                                if ($prog_desc_text ne "") {
+                                        $data_cache->{prog_cache}->{$url}->{desc} = [[ $prog_desc->as_text(), $opt->{lang} ]];
+                                        &log("  got desc '".$prog_desc->as_text()."'") if (defined $opt->{debug} && $opt->{debug} > 1);
+                                }
                         }
 
@@ -444 +480 @@
                 $data_cache->{prog_cache}->{$url}->{last_used} = time;
 
-                # augment prog details based on cache
                 foreach my $field ("desc", "subtitles", "video", "rating") {
                         if (defined $data_cache->{prog_cache}->{$url}->{$field}) {
-                                $prog->{$field} = $data_cache->{prog_cache}->{$url}->{$field};
-                        }
-                }
+                                $d->{progs}->[$i]->{$field} = $data_cache->{prog_cache}->{$url}->{$field};
+                        }
+                }
+
+                printf "DEBUG: prog $i details: ".Dumper($d->{progs}->[$i]) if (defined $opt->{debug});
 
                 unless ((defined $opt->{fast}) || ($was_in_cache)) {

status

@@ -13 +13 @@
 grabber         channelnsw_gov      0.6
 grabber         southerncross_website   0.04
+grabber         ten_website         1.00
 reconciler      reconciler_mk2      0.26
 postprocessor   imdb_augment_data   0.15