| 1 | #!/usr/bin/env perl |
|---|
| 2 | |
|---|
| 3 | # yahoo7_widget au_tv guide grabber - runs from "Shepherd" master grabber |
|---|
| 4 | # * written by ltd |
|---|
| 5 | # * uses yahoo7 widget for ABC/7/9/10/SBS (all they have) |
|---|
| 6 | # * when used in conjunction with Shepherd, shepherd can collect other channels |
|---|
| 7 | # using other grabbers |
|---|
| 8 | # * this does NOT use any config file - all settings are passed in from shepherd |
|---|
| 9 | |
|---|
| 10 | # changelog: |
|---|
| 11 | # 1.50 22sep06 added support for "shepherd" master grabber script |
|---|
| 12 | # 1.51 02oct06 --ready option |
|---|
| 13 | # 1.52 03oct06 split out yahoo7 grabber into its own grabber |
|---|
| 14 | # 1.54 16oct06 put date/cast/credits/year into correct xmltv fields |
|---|
| 15 | # 1.70 15dec06 sometimes there are holes in data; augment those from yahoo7web |
|---|
| 16 | # 1.78 04feb07 remove augmenting - micrograbbing from other grabbers |
|---|
| 17 | # can now fill those in |
|---|
| 18 | # 1.80 22feb07 bugfix: stop throwing away multiple category data |
|---|
| 19 | # 1.81 01mar07 more informative error on no connectivity |
|---|
| 20 | # 2.00 19jun10 moved to JSON data format |
|---|
| 21 | # 2.01 20jun10 add sub-title support in |
|---|
| 22 | # 2.10 22jun10 experimental: drop JSON module for cut-down Shepherd::json_pp |
|---|
| 23 | |
|---|
| 24 | use strict; |
|---|
| 25 | use warnings; |
|---|
| 26 | |
|---|
| 27 | my $progname = "yahoo7widget"; |
|---|
| 28 | my $version = "2.27"; |
|---|
| 29 | |
|---|
| 30 | use XMLTV; |
|---|
| 31 | use POSIX qw(strftime mktime); |
|---|
| 32 | use Getopt::Long; |
|---|
| 33 | use Data::Dumper; |
|---|
| 34 | use Shepherd::Common; |
|---|
| 35 | use Shepherd::json_pp; # use JSON; |
|---|
| 36 | |
|---|
| 37 | # |
|---|
| 38 | # some initial cruft |
|---|
| 39 | # |
|---|
| 40 | |
|---|
| 41 | my $script_start_time = time; |
|---|
| 42 | my %stats; |
|---|
| 43 | my $channels, my $opt_channels; |
|---|
| 44 | my $tv_guide; |
|---|
| 45 | my $input_xml; |
|---|
| 46 | my $d; |
|---|
| 47 | $| = 1; |
|---|
| 48 | |
|---|
| 49 | # map yahoo paytv names to what shepherd knows them as |
|---|
| 50 | my %paytv_fixups = ( qw{ |
|---|
| 51 | 111HITS 111Hits 13THSTREET2 13thSTREET2 A-PAC APAC Bio. Bio BloombergTELEVISION BloombergTelevision |
|---|
| 52 | BOOMERANG Boomerang CARTOONNETWORK CartoonNetwork BBCCBeebies Cbeebies CrimeInvestigation Crime DiscoveryHomeHealth DiscoveryHealth |
|---|
| 53 | DiscoveryTurboMAX2 DiscTurboMAX2 EUROSPORTNEWS Eurosportnews fashiontv FashionTV FOXSPORTS1HD FoxSports1HD FOXSPORTS2HD FoxSports2HD |
|---|
| 54 | FOXSPORTS3HD FoxSports3HD HITS2 111HITS2 TheLifeStyleChannel2 LifeStyle2 LifeStyleFOODChannel LifeStyleFOOD |
|---|
| 55 | LifeStyleFOODChannel2 LifestyleFOOD2 LifestyleYOU LifeStyleYOU LifeStyleYOU2 LifStyleYOU2 MTVClassics MTVClassic MTVLive MTVNLive |
|---|
| 56 | MTVLiveHD MTVNLiveHD NATGEOWILD NatGeoWild NATGEOWILDHD NatGeoWildHD NickJr. NickJr showtimepremiereHD ShowPremiereHD |
|---|
| 57 | SKYNEWSBUSINESS SKYNewsBusiness SKYNEWSAUSTRALIA SKYNewsNational SkyRacing SKYRacing SkyRacing2 SKYRacing2 STARPICS STARPICS1 |
|---|
| 58 | THECOMEDYCHANNEL ComedyChannel THECOMEDYCHANNEL2 Comedy2 TheWORLDMOVIESChannel WORLDMOVIES UniversalChannel Universal |
|---|
| 59 | } ); |
|---|
| 60 | |
|---|
| 61 | # |
|---|
| 62 | # parse command line |
|---|
| 63 | # |
|---|
| 64 | |
|---|
| 65 | my $opt; |
|---|
| 66 | $opt->{outputfile} = "output.xmltv"; # default |
|---|
| 67 | $opt->{days} = 7; # default |
|---|
| 68 | $opt->{lang} = "en"; |
|---|
| 69 | $opt->{region} = 94; |
|---|
| 70 | |
|---|
| 71 | GetOptions( |
|---|
| 72 | 'region=i' => \$opt->{region}, |
|---|
| 73 | 'days=i' => \$opt->{days}, |
|---|
| 74 | 'offset=i' => \$opt->{offset}, |
|---|
| 75 | 'paytvtimeoffset:s' => \$opt->{paytvoffset}, |
|---|
| 76 | 'channels_file=s' => \$opt->{channels_file}, |
|---|
| 77 | 'cachefile=s' => \$opt->{obsolete}, |
|---|
| 78 | 'output=s' => \$opt->{outputfile}, |
|---|
| 79 | 'fast' => \$opt->{fast}, |
|---|
| 80 | 'paytv' => \$opt->{paytv}, |
|---|
| 81 | 'warper' => \$opt->{warper}, |
|---|
| 82 | 'lang=s' => \$opt->{lang}, |
|---|
| 83 | 'obfuscate' => \$opt->{obfuscate}, |
|---|
| 84 | 'debug+' => \$opt->{debug}, |
|---|
| 85 | 'help' => \$opt->{help}, |
|---|
| 86 | 'verbose' => \$opt->{help}, |
|---|
| 87 | 'version' => \$opt->{version}, |
|---|
| 88 | 'ready' => \$opt->{version}, |
|---|
| 89 | 'desc' => \$opt->{desc}, |
|---|
| 90 | 'v' => \$opt->{help}); |
|---|
| 91 | |
|---|
| 92 | &help if (defined $opt->{help}); |
|---|
| 93 | |
|---|
| 94 | if (defined $opt->{version} || defined $opt->{desc}) { |
|---|
| 95 | printf "%s %s\n",$progname,$version; |
|---|
| 96 | printf "%s is a details-aware grabber that collects very high quality data (full title/subtitle/description/genre and year/cast/credits data) using the Yahoo7 website.",$progname if (defined $opt->{desc}); |
|---|
| 97 | exit(0); |
|---|
| 98 | } |
|---|
| 99 | |
|---|
| 100 | # set defaults |
|---|
| 101 | Shepherd::Common::set_default("debug", ($opt->{debug} * 2)) if (defined $opt->{debug}); |
|---|
| 102 | Shepherd::Common::set_default("webwarper", 1) if (defined $opt->{warper}); |
|---|
| 103 | Shepherd::Common::set_default("squid", 1) if (defined $opt->{obfuscate}); |
|---|
| 104 | Shepherd::Common::set_default("referer", "last"); |
|---|
| 105 | Shepherd::Common::set_default("retry_delay", 10); |
|---|
| 106 | Shepherd::Common::setup_ua('agent' => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-us)'); |
|---|
| 107 | |
|---|
| 108 | die "no channel file specified, see --help for instructions\n", unless (defined $opt->{channels_file}); |
|---|
| 109 | |
|---|
| 110 | $opt->{days} = 9 if $opt->{days} > 9; # no data beyond 9 days |
|---|
| 111 | |
|---|
| 112 | # |
|---|
| 113 | # go go go! |
|---|
| 114 | # |
|---|
| 115 | |
|---|
| 116 | my $starttime = $script_start_time; |
|---|
| 117 | |
|---|
| 118 | # correct $starttime to remove gap between local and Eastern States time |
|---|
| 119 | my $local_time_offset; |
|---|
| 120 | |
|---|
| 121 | if (!defined $opt->{paytvoffset} || !($opt->{paytvoffset} =~ m/(\+|-)\d{4}$/o)) { |
|---|
| 122 | $local_time_offset = POSIX::strftime("%z",localtime(time)); |
|---|
| 123 | } else { |
|---|
| 124 | $local_time_offset = $opt->{paytvoffset}; |
|---|
| 125 | } |
|---|
| 126 | |
|---|
| 127 | # yahoo7widget times are always localtime on Eastern States .. |
|---|
| 128 | $ENV{TZ}="Australia/Melbourne"; |
|---|
| 129 | my $melbourne_time_offset = POSIX::strftime("%z",localtime(time)); |
|---|
| 130 | my $time_offset = ((int(substr($melbourne_time_offset,1,2))-int(substr($local_time_offset,1,2)))*(60*60)) + |
|---|
| 131 | ((int(substr($melbourne_time_offset,3,2))-int(substr($local_time_offset,3,2)))*(60)); |
|---|
| 132 | $starttime -= $time_offset; |
|---|
| 133 | |
|---|
| 134 | my $endtime = $starttime + ($opt->{days} * 86400); |
|---|
| 135 | $starttime += (86400 * $opt->{offset}) if (defined $opt->{offset}); |
|---|
| 136 | |
|---|
| 137 | Shepherd::Common::log(sprintf "going to grab %d days%s of data into %s (%s%s) region %s", |
|---|
| 138 | $opt->{days}, |
|---|
| 139 | ($opt->{offset} ? " (skipping first $opt->{offset} days)" : ""), |
|---|
| 140 | $opt->{outputfile}, |
|---|
| 141 | (defined $opt->{fast} ? "with haste" : "slowly"), |
|---|
| 142 | (defined $opt->{warper} ? ", anonymously" : ""), |
|---|
| 143 | $opt->{region}); |
|---|
| 144 | |
|---|
| 145 | if (-r $opt->{channels_file}) { |
|---|
| 146 | local (@ARGV, $/) = ($opt->{channels_file}); |
|---|
| 147 | no warnings 'all'; eval <>; die "$@" if $@; |
|---|
| 148 | } else { |
|---|
| 149 | die "WARNING: channels file $opt->{channels_file} could not be read\n"; |
|---|
| 150 | } |
|---|
| 151 | |
|---|
| 152 | &check_for_paytv; |
|---|
| 153 | |
|---|
| 154 | $starttime -= ($starttime % $d->{fetch_interval}); # normalize time window to help yahoo's squid cache entries |
|---|
| 155 | for (my $currtime = $starttime; $currtime < $endtime; $currtime += $d->{fetch_interval}) { |
|---|
| 156 | # pace ourselves |
|---|
| 157 | if (($currtime != $starttime) && (!defined $opt->{fast})) { |
|---|
| 158 | my $sleeptimer = int(rand(5)) + 1; # sleep anywhere from 1 to 5 seconds |
|---|
| 159 | $stats{slept_for} += $sleeptimer; |
|---|
| 160 | sleep $sleeptimer; |
|---|
| 161 | } |
|---|
| 162 | |
|---|
| 163 | # get data |
|---|
| 164 | my $tries = 5; |
|---|
| 165 | my $url = sprintf "http://au.tv.yahoo.com/tv-guide/data/%d/%d/%d/%d/", |
|---|
| 166 | $opt->{region}, |
|---|
| 167 | ($d->{want_paytv} > 0 ? "168" : "0"), |
|---|
| 168 | $currtime, |
|---|
| 169 | ($currtime+$d->{fetch_interval}); |
|---|
| 170 | my ($data, $success, $status_msg, $bytes_fetched, $seconds_slept, $failed_attempts, $response) = |
|---|
| 171 | Shepherd::Common::get_url(url => $url, retries => ($tries-1)); |
|---|
| 172 | |
|---|
| 173 | $stats{failed_requests} += $failed_attempts; |
|---|
| 174 | $stats{slept_for} += $seconds_slept; |
|---|
| 175 | $stats{bytes_fetched} += $bytes_fetched; |
|---|
| 176 | |
|---|
| 177 | if ((!$data) || (!$success)) { |
|---|
| 178 | Shepherd::Common::log("Failed to fetch '$url' after $tries attempts."); |
|---|
| 179 | |
|---|
| 180 | # if its our first page, abort now |
|---|
| 181 | if ($currtime == $starttime) { |
|---|
| 182 | Shepherd::Common::log("Aborting: likely format change or blocked!"); |
|---|
| 183 | exit 10; |
|---|
| 184 | } |
|---|
| 185 | next; |
|---|
| 186 | } |
|---|
| 187 | |
|---|
| 188 | $stats{http_successful_requests}++; |
|---|
| 189 | &parse_json_data($data); |
|---|
| 190 | } |
|---|
| 191 | |
|---|
| 192 | &write_data; |
|---|
| 193 | Shepherd::Common::print_stats($progname, $version, $script_start_time, %stats); |
|---|
| 194 | exit(0); |
|---|
| 195 | |
|---|
| 196 | ###################################################################################################### |
|---|
| 197 | # help |
|---|
| 198 | |
|---|
| 199 | sub help |
|---|
| 200 | { |
|---|
| 201 | print<<EOF |
|---|
| 202 | $progname $version |
|---|
| 203 | |
|---|
| 204 | $0 [options] |
|---|
| 205 | options are as follows: |
|---|
| 206 | --region=N set region for where to collect data from (default: $opt->{region}) |
|---|
| 207 | --channels_file=file where to get channel data from (MANDATORY) |
|---|
| 208 | --days=N fetch 'n' days of data (default: $opt->{days}) |
|---|
| 209 | --output=file send xml output to file (default: "$opt->{outputfile}") |
|---|
| 210 | |
|---|
| 211 | --fast don't run slow - get data as quick as you can - not recommended |
|---|
| 212 | --debug increase debug level |
|---|
| 213 | --warper fetch data using WebWarper web anonymizer service |
|---|
| 214 | --obfuscate pretend to be a proxy servicing multiple clients |
|---|
| 215 | --lang=[s] set language of xmltv output data (default $opt->{lang}) |
|---|
| 216 | |
|---|
| 217 | --paytvtimeoffset apply an offset between current timezone and AEST for paytv channels |
|---|
| 218 | --paytvtimeoffset[="+HHMM"] apply a difference of +HHMM and AEST for paytv channels ie +1030 (+- is mandatory) |
|---|
| 219 | |
|---|
| 220 | EOF |
|---|
| 221 | ; |
|---|
| 222 | |
|---|
| 223 | exit(0); |
|---|
| 224 | } |
|---|
| 225 | |
|---|
| 226 | ###################################################################################################### |
|---|
| 227 | # parse it into $tv_guide->{$channel}->{data}->{$event_id}-> structures.. |
|---|
| 228 | |
|---|
| 229 | sub parse_json_data |
|---|
| 230 | { |
|---|
| 231 | my $data = shift; |
|---|
| 232 | |
|---|
| 233 | unless (($data) && (length($data) > 1)) { |
|---|
| 234 | print STDERR "ERROR: no data to parse.\n"; |
|---|
| 235 | return; |
|---|
| 236 | } |
|---|
| 237 | |
|---|
| 238 | # my $decoded_data = decode_json($data); |
|---|
| 239 | my $decoded_data = JSON::cut_down_PP::decode_json($data); |
|---|
| 240 | |
|---|
| 241 | foreach my $item (@{($decoded_data->{'tv'}->[0]->{'item'})}) { |
|---|
| 242 | my $event_id = $item->{'event_id'}; |
|---|
| 243 | |
|---|
| 244 | # mandatory fields |
|---|
| 245 | my $event_start = $item->{'event_date'}->[0]->{timestamp}; |
|---|
| 246 | my $event_end = $item->{'end_date'}->[0]->{timestamp}; |
|---|
| 247 | |
|---|
| 248 | if ($event_start < 10) { |
|---|
| 249 | $stats{progs_with_invalid_start}++; |
|---|
| 250 | printf "WARNING: programme with event_id '$event_id' had an invalid start time of '$event_start'; skipped\n"; |
|---|
| 251 | next; |
|---|
| 252 | } |
|---|
| 253 | if ($event_end < 10) { |
|---|
| 254 | $stats{progs_with_invalid_end}++; |
|---|
| 255 | printf "WARNING: programme with event_id '$event_id' had an invalid end time of '$event_end'; skipped\n"; |
|---|
| 256 | next; |
|---|
| 257 | } |
|---|
| 258 | |
|---|
| 259 | my $channel = $item->{'co_v_short'}; |
|---|
| 260 | |
|---|
| 261 | if (ref($channel) ne "") { # paytv channels |
|---|
| 262 | $channel = $channel->[0]->{0}; |
|---|
| 263 | $channel =~ s/(\s|\[|\]|\+|\&)//g; |
|---|
| 264 | $channel = $paytv_fixups{$channel} if (defined $paytv_fixups{$channel}); |
|---|
| 265 | |
|---|
| 266 | $event_start -= $time_offset if (defined $opt->{paytvoffset}); |
|---|
| 267 | $event_end -= $time_offset if (defined $opt->{paytvoffset}); |
|---|
| 268 | } |
|---|
| 269 | |
|---|
| 270 | # Hack; assuming this is a temporary typo |
|---|
| 271 | $channel = 'ABC News 24' if ($channel eq 'ABCNews24'); |
|---|
| 272 | $channel = 'WIN GEM' if ($channel eq 'WINGEM'); |
|---|
| 273 | $channel = '31 Digital' if ($channel eq '31Digital'); |
|---|
| 274 | $channel = 'ABC2 / ABC4' if ($channel eq 'ABC2/ABC4'); |
|---|
| 275 | |
|---|
| 276 | # Temp hack Oct 2010 when Seven-related channels gave bad data |
|---|
| 277 | # if (grep ($channel eq $_, ('Seven', 'Prime', '7mate', '7TWO', 'Prime HD', 'Seven HD'))) |
|---|
| 278 | # { |
|---|
| 279 | # &Shepherd::Common::log("ignoring unreliable channel '$channel'"); |
|---|
| 280 | # $d->{ignored_channels}->{$channel} = 1; |
|---|
| 281 | # $stats{skipped_channels}++; |
|---|
| 282 | # next; |
|---|
| 283 | # } |
|---|
| 284 | |
|---|
| 285 | if ((!defined $channels->{$channel}) && (!defined $opt_channels->{$channel})) { |
|---|
| 286 | if (!defined $d->{ignored_channels}->{$channel}) { |
|---|
| 287 | $d->{ignored_channels}->{$channel} = 1; |
|---|
| 288 | $stats{skipped_channels}++; |
|---|
| 289 | Shepherd::Common::log("ignoring unwanted channel '$channel'"); |
|---|
| 290 | } |
|---|
| 291 | next; |
|---|
| 292 | } |
|---|
| 293 | |
|---|
| 294 | |
|---|
| 295 | $event_id = $event_start.":".$event_end.":".$event_id; # event_id actually isn't unique - so make it so |
|---|
| 296 | |
|---|
| 297 | $stats{programmes}++; |
|---|
| 298 | $stats{duplicate_programmes}++ if ($tv_guide->{$channel}->{data}->{$event_id}); |
|---|
| 299 | |
|---|
| 300 | # store it in the correct XMLTV schema! |
|---|
| 301 | $tv_guide->{$channel}->{data}->{$event_id}->{'channel'} = $channels->{$channel} if (defined $channels->{$channel}); |
|---|
| 302 | $tv_guide->{$channel}->{data}->{$event_id}->{'channel'} = $opt_channels->{$channel} if (defined $opt_channels->{$channel}); |
|---|
| 303 | |
|---|
| 304 | $tv_guide->{$channel}->{data}->{$event_id}->{'start'} = POSIX::strftime("%Y%m%d%H%M", localtime($event_start)); |
|---|
| 305 | $tv_guide->{$channel}->{data}->{$event_id}->{'stop'} = POSIX::strftime("%Y%m%d%H%M", localtime($event_end)); |
|---|
| 306 | |
|---|
| 307 | # Hack for what seems to be a bug in the datasource: |
|---|
| 308 | # i.e. titles/sub-titles of "Law &Order" rather than "Law & Order". |
|---|
| 309 | $item->{title}->[0]->{0} =~ s/ \&(?=\S)/ \& /g if (defined $item->{title}); |
|---|
| 310 | $item->{subtitle}->[0]->{0} =~ s/ \&(?=\S)/ \& /g if (defined $item->{subtitle}); |
|---|
| 311 | |
|---|
| 312 | $tv_guide->{$channel}->{data}->{$event_id}->{'title'} = [[ $item->{title}->[0]->{0}, $opt->{lang} ]] if (defined $item->{title}); |
|---|
| 313 | $tv_guide->{$channel}->{data}->{$event_id}->{'desc'} = [[ $item->{description_1}, $opt->{lang} ]] if (defined $item->{description_1}); |
|---|
| 314 | $tv_guide->{$channel}->{data}->{$event_id}->{'sub-title'} = [[ $item->{subtitle}->[0]->{0}, $opt->{lang} ]] if (defined $item->{subtitle}); |
|---|
| 315 | |
|---|
| 316 | my %genre, my $category, my $category2; |
|---|
| 317 | $genre{movie} = 1 if ((defined $item->{show_type}) && ($item->{show_type} eq "Movie")); |
|---|
| 318 | $genre{live} = 1 if (defined $item->{live}); |
|---|
| 319 | $category = $item->{genre} if (defined $item->{genre}); |
|---|
| 320 | if ((defined $item->{'category'}) && (defined $item->{'category'}->[0]->{'item'}->[0]->{0})) { |
|---|
| 321 | my $category2 = $item->{'category'}->[0]->{'item'}->[0]->{0}; |
|---|
| 322 | $category = $category2 if (!defined $category); |
|---|
| 323 | $genre{$category2} = 1; |
|---|
| 324 | } |
|---|
| 325 | $tv_guide->{$channel}->{data}->{$event_id}->{'category'} = [ &Shepherd::Common::generate_category($item->{title}->[0]->{0}, $category, %genre) ]; |
|---|
| 326 | Shepherd::Common::log(sprintf "category: chose '%s' from '%s' and '%s' for '%s'", |
|---|
| 327 | $tv_guide->{$channel}->{data}->{$event_id}->{'category'}->[0]->[0], $category, |
|---|
| 328 | join(', ',keys %genre), $item->{title}->[0]->{0}) if (defined $opt->{debug}); |
|---|
| 329 | |
|---|
| 330 | $tv_guide->{$channel}->{data}->{$event_id}->{'country'} = [ map([$_,$opt->{lang}], split(/\//, $item->{country})) ] if (defined $item->{country}); |
|---|
| 331 | $tv_guide->{$channel}->{data}->{$event_id}->{'premiere'} = [ 'premiere', $opt->{lang} ] if (defined $item->{'premiere'}); |
|---|
| 332 | |
|---|
| 333 | my $rating = ""; |
|---|
| 334 | $rating .= $item->{rating} if (defined $item->{'rating'}); |
|---|
| 335 | $rating .= " ".lc($item->{warnings}) if (defined $item->{'warnings'}); |
|---|
| 336 | $tv_guide->{$channel}->{data}->{$event_id}->{'rating'} = [[ $rating, 'ABA', undef ]] if $rating ne ""; |
|---|
| 337 | |
|---|
| 338 | $tv_guide->{$channel}->{data}->{$event_id}->{'credits'}{'actor'} = [ split(/, /, $item->{main_cast}) ] if (defined $item->{main_cast}); |
|---|
| 339 | $tv_guide->{$channel}->{data}->{$event_id}->{'credits'}{'director'} = [ split(/, /, $item->{director}) ] if (defined $item->{director}); |
|---|
| 340 | $tv_guide->{$channel}->{data}->{$event_id}->{'credits'}{'writer'} = [ split(/, /, $item->{writer}) ] if (defined $item->{writer}); # unseen |
|---|
| 341 | $tv_guide->{$channel}->{data}->{$event_id}->{'date'} = $item->{year_released} if (defined $item->{year_released}); |
|---|
| 342 | $tv_guide->{$channel}->{data}->{$event_id}->{'previously-shown'} = { } if (defined $item->{repeat}); |
|---|
| 343 | $tv_guide->{$channel}->{data}->{$event_id}->{'subtitles'} = [ { 'type' => 'teletext' } ] if (defined $item->{captions}); |
|---|
| 344 | $tv_guide->{$channel}->{data}->{$event_id}->{'last-chance'} = [ 'final', $opt->{lang} ] if (defined $item->{final}); |
|---|
| 345 | |
|---|
| 346 | my %video_details; |
|---|
| 347 | if ((defined $item->{colour}) && ($item->{colour} == 1)) { |
|---|
| 348 | $video_details{'colour'} = 1; |
|---|
| 349 | $tv_guide->{$channel}->{data}->{$event_id}->{'video'} = \%video_details; |
|---|
| 350 | } |
|---|
| 351 | |
|---|
| 352 | $tv_guide->{$channel}->{data}->{$event_id}->{'length'} = ($item->{running_time} * 60) if (defined $item->{running_time}); |
|---|
| 353 | $tv_guide->{$channel}->{data}->{$event_id}->{'language'} = [ $item->{language}, $opt->{lang} ] if (defined $item->{language}); |
|---|
| 354 | |
|---|
| 355 | $d->{seen_progs}->{$channel}++; |
|---|
| 356 | |
|---|
| 357 | # print out any unused fields |
|---|
| 358 | foreach my $field (keys %{($item)}) { |
|---|
| 359 | next if ($field =~ /(event_id|co_v_short|event_date|end_date|title|subtitle|description_1|show_type|category|country|premiere|rating|warnings|main_cast|director|writer|year_released|repeat|captions|final|colour|running_time|language|genre|live)/); # parsed fields |
|---|
| 360 | next if ($field =~ /(program_id|series_id|genre_id|venue_id|y7_url|highlight)/); # ignored fields |
|---|
| 361 | next if (defined $d->{ignored_field}->{$field}); |
|---|
| 362 | $d->{ignored_field}->{$field} = 1; |
|---|
| 363 | Shepherd::Common::log("ignoring unknown field '$field'"); |
|---|
| 364 | } |
|---|
| 365 | } |
|---|
| 366 | } |
|---|
| 367 | |
|---|
| 368 | ###################################################################################################### |
|---|
| 369 | |
|---|
| 370 | sub write_data |
|---|
| 371 | { |
|---|
| 372 | my %writer_args = ( encoding => 'ISO-8859-1' ); |
|---|
| 373 | if ($opt->{outputfile}) { |
|---|
| 374 | my $fh = new IO::File(">$opt->{outputfile}") or die "can't open $opt->{outputfile}: $!"; |
|---|
| 375 | $writer_args{OUTPUT} = $fh; |
|---|
| 376 | } |
|---|
| 377 | |
|---|
| 378 | my $writer = new XMLTV::Writer(%writer_args); |
|---|
| 379 | |
|---|
| 380 | $writer->start |
|---|
| 381 | ( { 'source-info-name' => "$progname $version", |
|---|
| 382 | 'generator-info-name' => "$progname $version"} ); |
|---|
| 383 | |
|---|
| 384 | for my $channel (sort keys %{$channels}) { |
|---|
| 385 | $writer->write_channel( {'display-name' => [[ $channel, $opt->{lang} ]], 'id' => $channels->{$channel}} ) |
|---|
| 386 | if (defined $d->{seen_progs}->{$channel}); |
|---|
| 387 | } |
|---|
| 388 | for my $channel (sort keys %{$opt_channels}) { |
|---|
| 389 | $writer->write_channel( {'display-name' => [[ $channel, $opt->{lang} ]], 'id' => $opt_channels->{$channel}} ) |
|---|
| 390 | if (defined $d->{seen_progs}->{$channel}); |
|---|
| 391 | } |
|---|
| 392 | |
|---|
| 393 | my $abc2_eariest_start = "300000000000"; |
|---|
| 394 | |
|---|
| 395 | for my $channel (sort keys %{($d->{seen_progs})}) { |
|---|
| 396 | for my $event_id (sort {$a cmp $b} keys %{($tv_guide->{$channel}->{data})}) { |
|---|
| 397 | my $show = $tv_guide->{$channel}->{data}->{$event_id}; |
|---|
| 398 | Shepherd::Common::cleanup($show); |
|---|
| 399 | $writer->write_programme($show); |
|---|
| 400 | |
|---|
| 401 | $abc2_eariest_start = $show->{'start'} |
|---|
| 402 | if ($channel eq "ABC2" && $abc2_eariest_start > $show->{'start'}); |
|---|
| 403 | } |
|---|
| 404 | } |
|---|
| 405 | |
|---|
| 406 | # check if abc2 has a gap on the first day when the station is closed |
|---|
| 407 | if ($abc2_eariest_start != "300000000000" && (!defined $opt->{offset}) && defined $channels->{ABC2} && |
|---|
| 408 | $abc2_eariest_start > POSIX::strftime("%Y%m%d%H%M", localtime($starttime))) { |
|---|
| 409 | |
|---|
| 410 | # create 7am today |
|---|
| 411 | my @timeattr = localtime($script_start_time); # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst |
|---|
| 412 | $timeattr[0] = 0; # zero seconds |
|---|
| 413 | $timeattr[1] = 0; # min |
|---|
| 414 | $timeattr[2] = 7; # hours 7am |
|---|
| 415 | my $time7am = mktime(@timeattr); |
|---|
| 416 | $time7am -= $time_offset; |
|---|
| 417 | my $xmltime7am = POSIX::strftime("%Y%m%d%H%M", localtime($time7am)); |
|---|
| 418 | |
|---|
| 419 | if (($starttime < $time7am) && ($abc2_eariest_start == $xmltime7am)) |
|---|
| 420 | { |
|---|
| 421 | my $show; |
|---|
| 422 | $show->{'channel'} = $channels->{ABC2}; |
|---|
| 423 | $show->{'title'} = [[ "Station Close Guess", $opt->{lang} ]]; |
|---|
| 424 | $show->{'start'} = POSIX::strftime("%Y%m%d%H%M", localtime($starttime)); |
|---|
| 425 | $show->{'stop'} = $abc2_eariest_start; |
|---|
| 426 | |
|---|
| 427 | Shepherd::Common::cleanup($show); |
|---|
| 428 | $writer->write_programme($show); |
|---|
| 429 | } |
|---|
| 430 | } |
|---|
| 431 | |
|---|
| 432 | $writer->end(); |
|---|
| 433 | } |
|---|
| 434 | |
|---|
| 435 | ###################################################################################################### |
|---|
| 436 | |
|---|
| 437 | sub check_for_paytv |
|---|
| 438 | { |
|---|
| 439 | $d->{want_paytv} = 0; |
|---|
| 440 | foreach my $ch ( qw( |
|---|
| 441 | 111Hits 111HITS2 13THSTREET 13thSTREET2 3DChannel ACC AdultsOnly960 AIR AnimalPlanet AntennaGreek AntennaPacific |
|---|
| 442 | APAC Arena Arena2 Aurora BBCKnowledge BBCKnowledgeHD BBCWorldNews Bio BloombergTelevision Boomerang CartoonNetwork |
|---|
| 443 | Cbeebies ChannelV CMC CNBC CNN Comedy2 ComedyChannel Crime Discovery2 DiscoveryChannel DiscoveryHD DiscoveryHealth |
|---|
| 444 | DiscoveryRealTime DiscoveryScience DiscoveryTravel DiscoveryTurboMAX DiscoveryWorldHD DiscTurboMAX2 DisneyChannel |
|---|
| 445 | E!Entertainment ERTGreek ESPN ESPNHD Eurosport Eurosportnews EXPO FashionTV FMC FOX8 FOX82 FOX8HD FOXClassics |
|---|
| 446 | FOXClassics2 FOXNews FOXSPORTS1 FoxSports1HD FOXSPORTS2 FoxSports2HD FOXSPORTS3 FoxSports3HD FOXSPORTSNews |
|---|
| 447 | FoxtelBoxOffice FUELTV Hallmark HOWTOChannel KidsCo LifeStyle2 LifeStyleFOOD LifestyleFOOD2 LifeStyleYOU LifStyleYOU2 |
|---|
| 448 | MAX MOVIEEXTRA MOVIEGREATS MOVIEONE MOVIEONEHD MOVIETWO MTV MTVClassic MTVHits MTVNLive MTVNLiveHD NatGeo2 |
|---|
| 449 | NatGeoAdventure NatGeoAdventure2 NatGeoHD NATGEOWILD NatGeoWild NatGeoWildHD NationalGeographic Nickelodeon |
|---|
| 450 | NickJr NITV Ovation PlayhouseDisney RAIITALIA SCIFI SCIFI2 showcase showcaseHD showcasetwo ShowPremiereHD |
|---|
| 451 | showtimeaction showtimeactionHD showtimecomedy showtimedrama showtimepremiere showtimetwo SkyNewsAustralia SkyNewsBusiness |
|---|
| 452 | SKYNewsBusiness SKYNewsNational SkyRacing SKYRacing SKYRacing2 SKYRacingWorld SPEED SPEEDHD STARPICS1 STARPICS1HD STARPICS2 |
|---|
| 453 | STARPICS2HD STVDIO TCM TheHistoryChannel TheLifeStyleChannel TheStyleNetwork TheWeatherChannel TLC TLC2 TSN TV1 TV12 |
|---|
| 454 | TVE TVN TVSN UKTV UKTV2 UKTVHD Universal VHits W W2 WHD WORLDMOVIES |
|---|
| 455 | )) { |
|---|
| 456 | $d->{want_paytv}++ if ((exists $channels->{$ch} || exists $opt_channels->{$ch})); |
|---|
| 457 | } |
|---|
| 458 | |
|---|
| 459 | $d->{want_paytv}++ if (defined $opt->{paytv}); |
|---|
| 460 | if ($d->{want_paytv} > 0) { |
|---|
| 461 | Shepherd::Common::log("going to gather paytv channels as we have ".$d->{want_paytv}." paytv channels matching."); |
|---|
| 462 | Shepherd::Common::log("offsetting paytv channels by ".($time_offset/60)." minutes due to --paytvoffset") if (defined $opt->{paytvoffset}); |
|---|
| 463 | $d->{fetch_interval} = (3 * 3600); # 3 hours (for now) |
|---|
| 464 | } else { |
|---|
| 465 | $d->{fetch_interval} = (6 * 3600); # 6 hours |
|---|
| 466 | } |
|---|
| 467 | } |
|---|