Context Navigation

abc_website

Revision 1390, 31.5 kB (checked in by max, 8 months ago)
shepherd 1.6.0: Change shebang lines from '/usr/bin/perl' to '/usr/bin/env perl', which is more portable (i.e. works on OSX).
Property svn:executable set to ``*

Line
1	#!/usr/bin/env perl
2
3	# ABC1/ABC2/ABC3/ABC News 24 au_tv guide grabber - runs from "Shepherd" master grabber
4	# * written by ltd
5	# * uses ABC website for ABC1 ABC2 ABC3 ABC News 24 data
6	# * when used in conjunction with Shepherd, shepherd can collect other channels
7	# using other grabbers
8	# * this does NOT use any config file - all settings are passed in from shepherd
9
10	# changelog:
11	# 1.50 22sep06 added support for "shepherd" master grabber script
12	# 1.51 02oct06 --ready option
13	# 1.52 03oct06 split out abc grabber into its own grabber
14	# 1.55 09oct06 formalize --cheap option
15	# 1.56 20oct06 misc cleanups
16	# 1.60 11nov06 fix midday time calculation
17	# 1.70 16nov06 also use "printable" TV guide to determine 'station close'
18	# 2.00 23nov06 simplified
19	# 3.00 23may07 merged abc1/abc2 together again, abc_website now does both!
20	# 3.04 12jun07 abc website format change
21	# 3.23 01dec09 added abc3
22	# 4.00 14aug10 new website format, much redesign, added abc4/abchd/"ABC News 24"
23
24	use strict;
25	use warnings;
26
27	my $progname = "abc_website";
28	my $version = "4.11";
29
30	use XMLTV;
31	use POSIX qw(strftime mktime);
32	use Getopt::Long;
33	use HTML::TreeBuilder;
34	use Shepherd::Common;
35
36	#
37	# constants
38	#
39	my $urls;
40	$urls->{station_close}->{"ABC1"} = "http://www.abc.net.au/tv/guide/abc1-7day-guide.htm";
41	$urls->{station_close}->{"ABC2 / ABC4"} = "http://www.abc.net.au/tv/guide/abc2-7day-guide.htm";
42	$urls->{station_close}->{"ABC3"} = "http://www.abc.net.au/tv/guide/abc3-7day-guide.htm";
43	$urls->{station_close}->{"ABC News 24"} = "http://www.abc.net.au/tv/guide/abcnews24-7day-guide.htm";
44	$urls->{guide}->{"ABC"} = "http://www.abc.net.au/tv/guide/all/";
45
46	#
47	# some initial cruft
48	#
49
50	my $script_start_time = time;
51	my %stats;
52	my $channels, my $opt_channels, my $gaps;
53	my $tv_guide;
54	my $data_cache;
55	my $override_settings = { };
56	my $writer;
57	$\| = 1;
58
59	my %webcache;
60	my %chanID_to_divID = (
61	'ABC1' => 'epgAbc1',
62	'ABC2 / ABC4' => 'epgAbc2',
63	'ABC3' => 'epgAbc3',
64	'ABC News 24' => 'epgAbc4',
65	);
66
67	#
68	# parse command line
69	#
70
71	my $opt_days = 7; # default
72	my $opt_offset = 0; # default
73	my $opt_outputfile = $progname.".xmltv"; # default
74	my $opt_configfile = $progname.".conf"; # ignored
75	my $opt_cache_file = $progname.".storable.cache";
76	my $opt_channels_file= "";
77	my $opt_gaps_file= "";
78	my $opt_no_cache = 0;
79	my $opt_cheap = 0;
80	my $opt_fast = 0;
81	my $opt_do_extra_days = 0;
82	my $opt_no_hdtv_flag = 0;
83	my $opt_set = "";
84	my $opt_help = 0;
85	my $opt_version = 0;
86	my $opt_desc = 0;
87	my $debug = 0;
88	my $lang = "en";
89	my $region = 94;
90	my $time_offset = 0;
91	my $ignored_setting;
92
93	GetOptions(
94	'region=i' => \$region,
95	'days=i' => \$opt_days,
96	'offset=i' => \$opt_offset,
97	'timezone=s' => \$ignored_setting,
98	'channels_file=s' => \$opt_channels_file,
99	'gaps_file=s' => \$opt_gaps_file,
100	'output=s' => \$opt_outputfile,
101	'config-file=s' => \$opt_configfile,
102	'cache-file=s' => \$opt_cache_file,
103	'do-extra-days' => \$opt_do_extra_days,
104	'fast' => \$opt_fast,
105	'no-cache' => \$opt_no_cache,
106	'cheap' => \$opt_cheap,
107	'debug+' => \$debug,
108	'warper' => \$ignored_setting,
109	'lang=s' => \$lang,
110	'obfuscate' => \$ignored_setting,
111	'no-retry' => \$ignored_setting,
112	'set=s' => \$opt_set,
113	'no_hdtv_flag' => \$opt_no_hdtv_flag,
114	'help' => \$opt_help,
115	'verbose' => \$opt_help,
116	'version' => \$opt_version,
117	'ready' => \$opt_version,
118	'desc' => \$opt_desc,
119	'v' => \$opt_help);
120
121	&help if ($opt_help);
122
123	if ($opt_version \|\| $opt_desc) {
124	printf "%s %s\n",$progname,$version;
125	printf "%s is a details-aware grabber that collects decent quality data using the ABC website for ABC1/ABC2/4/ABC3/ABC News 24.",$progname if $opt_desc;
126	exit(0);
127	}
128
129	&set_override if ($opt_set ne "");
130
131	die "no channel file specified, see --help for instructions\n", if ($opt_channels_file eq "");
132
133	# set defaults
134	&Shepherd::Common::set_default("debug", (defined $debug ? ($debug * 2) : 0));
135	&Shepherd::Common::set_defaults(stats => \%stats, referer => "last");
136
137	#
138	# go go go!
139	#
140
141	&read_cache if ($opt_no_cache == 0);
142
143	# figure out region, "netw" is national
144	my $abc_region = lc(Shepherd::Common::which_state($region));
145	$urls->{guide}->{ABC} .= $abc_region;
146
147	# read channels file
148	die "WARNING: channels file $opt_channels_file could not be read: $!\n" if (!-r $opt_channels_file);
149	local (@ARGV, $/) = ($opt_channels_file);
150	no warnings 'all'; eval <>; die "$@" if $@;
151
152	die "nothing to do; none of ABC1, ABC2 / ABC4, ABC3 or ABC News 24 in channels lineup!\n"
153	if ( (!defined $channels->{"ABC1"}) &&
154	(!defined $channels->{"ABC2 / ABC4"}) &&
155	(!defined $channels->{"ABC3"}) &&
156	(!defined $channels->{"ABC News 24"}) &&
157	1);
158
159	# check XMLTV version for HDTV compatability
160	my @xmltv_version = split(/\./,$XMLTV::VERSION);
161	if (($xmltv_version[0] <= 0) && ($xmltv_version[1] <= "5") && ($xmltv_version[2] <= "43")) {
162	&log("XMLTV version ".$XMLTV::VERSION." too old to support HDTV flags. Disabling HDTV flags.");
163	$opt_no_hdtv_flag = 1;
164	$stats{disabled_hdtv_flag}++;
165	}
166
167	&log(sprintf "Going to %s%s %s%d%s days%s of data for ABC1(%s), ABC2 / ABC4(%s), ABC3(%s) or ABC News 24(%s) into %s (%s)",
168	($opt_gaps_file ne "" ? "micro-gap " : ""),
169	($opt_cheap ? "verify (cache-validate)" : "grab"),
170	($opt_do_extra_days ? "somewhere between " : ""),
171	$opt_days,
172	($opt_do_extra_days ? " to 28" : ""),
173	($opt_offset ? " (skipping first $opt_offset days)" : ""),
174	(defined $channels->{"ABC1"} ? "yes:$abc_region" : "no"),
175	(defined $channels->{"ABC2 / ABC4"} ? "yes" : "no"),
176	(defined $channels->{"ABC3"} ? "yes" : "no"),
177	(defined $channels->{"ABC News 24"} ? "yes" : "no"),
178	$opt_outputfile,
179	($opt_no_cache ? "without caching" : "with caching"));
180
181	# if just filling in microgaps, parse gaps
182	if ($opt_gaps_file ne "") {
183	die "WARNING: gaps_file $opt_gaps_file could not be read: $!\n" if (!-r $opt_gaps_file);
184	local (@ARGV, $/) = ($opt_gaps_file);
185	no warnings 'all'; eval <>; die "$@" if $@;
186	}
187
188	my %writer_args = ( encoding => 'ISO-8859-1' );
189	my $fh = new IO::File(">$opt_outputfile") \|\| die "can't open $opt_outputfile: $!";
190	$writer_args{OUTPUT} = $fh;
191
192	$writer = new XMLTV::Writer(%writer_args);
193	$writer->start( { 'source-info-name' => "$progname $version", 'generator-info-name' => "$progname $version"} );
194
195	foreach my $ch ("ABC1","ABC2 / ABC4","ABC3","ABC News 24") {
196	$writer->write_channel( { 'display-name' => [[ $ch, $lang ]], 'id' => $channels->{$ch} } ) if (defined $channels->{$ch});
197	}
198
199	foreach my $ch ("ABC1","ABC2 / ABC4", "ABC3","ABC News 24") {
200	&get_station_close($ch, $urls->{station_close}->{$ch});
201	&get_abc_data($channels->{$ch}, $urls->{guide}->{ABC},$ch);
202	}
203
204	&write_cache if ($opt_no_cache == 0);
205
206	$writer->end;
207
208	&Shepherd::Common::print_stats($progname, $version, $script_start_time, %stats);
209	exit(0);
210
211	######################################################################################################
212	# help
213
214	sub help
215	{
216	print<<EOF
217	$progname $version
218
219	options are as follows:
220	--help show these help options
221	--days=N fetch 'n' days of data (default: $opt_days)
222	--output=file send xml output to file (default: "$opt_outputfile")
223	--config-file=file (ignored - historically used by grabbers not not this one)
224	--no-cache don't use a cache to optimize (reduce) number of web queries
225	--cheap validate contents of cache - fetch summary only, not details
226	--cache-file=file where to store cache (default "$opt_cache_file")
227	--fast don't run slow - get data as quick as you can - not recommended
228	--debug increase debug level
229	--do-extra-days fetch extra (21 days) from ABC website
230	--lang=[s] set language of xmltv output data (default $lang)
231	--no_hdtv_flag don't flag HDTV programmes
232
233	--region=N set region for where to collect data from (default: $region)
234	--channels_file=file where to get channel data from (if not set manually)
235
236	--set (option):(1/0) setting override options (1=enable, 0=disable)
237	do_extra_days:1/0 enable/disable fetching up to 24 days
238	fast:1/0 enable/disable extra-fast grab speed (not recommended)
239	debug:1/0 enable/disable debugging
240
241	EOF
242	;
243
244	exit(0);
245	}
246
247	######################################################################################################
248
249	sub set_override
250	{
251	&read_cache;
252	my ($setting, $val) = split(/:/,$opt_set);
253
254	die "--set format is (setting):(value) where value is 0 for disable, 1 for enable.\n"
255	if (($val ne "0") && ($val ne "1"));
256
257	die "unknown '--set' parameter '$setting', see --help for details.\n"
258	if (($setting ne "do_extra_days") &&
259	($setting ne "fast") &&
260	($setting ne "debug"));
261
262	$override_settings->{$setting} = $val;
263	printf "%s: override parameter %s: %s\n", $progname, $setting, ($val eq "0" ? "disabled" : "enabled");
264
265	&write_cache;
266	exit(0);
267	}
268
269	######################################################################################################
270	# populate cache
271
272	sub read_cache
273	{
274	my $store = Shepherd::Common::read_cache(\$opt_cache_file);
275
276	if ($store) {
277	$data_cache = $store->{data_cache};
278	$override_settings = $store->{override_settings};
279
280	# apply settings overrides
281	$opt_do_extra_days = 1 if ((defined $override_settings->{do_extra_days}) && ($override_settings->{do_extra_days} == 1));
282	$opt_fast = 1 if ((defined $override_settings->{fast}) && ($override_settings->{fast} == 1));
283	$debug = 1 if ((defined $override_settings->{debug}) && ($override_settings->{debug} > 0));
284	}
285	}
286
287	######################################################################################################
288	# write out updated cache
289
290	sub write_cache
291	{
292	# cleanup old entries from cache
293	for my $cache_key (keys %{$data_cache}) {
294	my ($starttime, @rest) = split(/,/,$cache_key);
295	if ($starttime < ($script_start_time - (22460*60))) {
296	delete $data_cache->{$cache_key};
297	$stats{removed_items_from_cache}++;
298	}
299	}
300
301	my $store;
302	$store->{data_cache} = $data_cache;
303	$store->{override_settings} = $override_settings;
304	Shepherd::Common::write_cache($opt_cache_file, $store);
305	}
306
307	######################################################################################################
308
309	sub get_abc_data
310	{
311	my ($xmlid,$urlbase,$chan_id) = @_;
312	my $try_to_add_abc_detail;
313	my $unprocessed_programmes = 0;
314	my $stop_fetching = 0;
315	my @unprocessed_progname, my @unprocessed_starttime, my @unprocessed_url, my @unprocessed_day;
316
317	my @gap_s, my @gap_e;
318
319	$opt_days = 28 if ($opt_do_extra_days);
320
321	# guide pages start at 4:30am so grab day before
322	my $daynum = $opt_offset - 1;
323	my $days_left = $opt_days - $opt_offset + 1;
324
325	my @timeattr = localtime($script_start_time); # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
326	$timeattr[0] = 0; # zero seconds
327	$timeattr[3] += $daynum - 1; # day
328
329	DAYS: while ($days_left > 0) {
330	$days_left--;
331	$daynum++;
332
333	# guide page starts at 4:30am
334	$timeattr[1] = 30; # minutes
335	$timeattr[2] = 4; # hours 4:30am
336	$timeattr[3]++; # day
337	my $currtime = mktime(@timeattr);
338
339	if ($opt_gaps_file ne "") { # micro-gap mode!
340	my $found_gap_match = 0;
341
342	if ((defined $gaps) && (defined $gaps->{$chan_id})) {
343	foreach my $g (@{($gaps->{$chan_id})}) {
344	my ($s, $e) = split(/-/,$g);
345	if (($s >= $currtime) && ($s <= ($currtime+86400))) {
346	$found_gap_match++;
347	push(@gap_s,$s);
348	push(@gap_e,$e);
349	printf "including day %d channel '%s' gap start %d, gap end %d\n",
350	$daynum, $chan_id, $s, $e if $debug;
351	}
352	}
353	}
354	next if (!$found_gap_match); # no gaps for this day - skip!
355	}
356
357	# fetch this URL once, and cache it for later
358	# TODO: maybe we should cache the tree, to save processing
359	# at the expense of memory
360	my $url = sprintf "%s/%s.htm",$urlbase, POSIX::strftime("%Y%m/%Y%m%d",localtime($currtime));
361
362	my $tries = ($daynum > 7 ? 1 : 5);
363	&log((sprintf "Fetching %s summary data: day %d of %d",
364	$xmlid, $daynum, $opt_days ));
365	my $data=$webcache{$url};
366	if (defined $data) {
367	&log("reusing previously fetched data for $url");
368	} else {
369	$data = Shepherd::Common::get_url(url => $url, retries => ($tries-1));
370	$webcache{$url}=$data;
371	}
372	my $tree;
373	$tree = HTML::TreeBuilder->new_from_content($data) if ($data);
374
375	if (!defined $tree) {
376	&log("failed to fetch $url after $tries attempts; skipping");
377
378	die "couldn't fetch first daily page after $tries attempts, network is probably down. aborting!"
379	if ((!defined $stats{abc_daily_pages}) \|\| ($stats{abc_daily_pages} == 0));
380
381	if ($daynum > 7) {
382	&log("failed to fetch $url, assuming we only have $daynum days..");
383	$days_left = 0;
384	}
385	next;
386	}
387
388	my $seen_programmes = 0;
389	my $seen_pm = 0;
390
391	# goes like this, for each channel:
392	#
393	# <div id="epgAbc1" class="epgRowWrap">
394	# <li style="width: 955px; ">
395	# <span class="itemTitle">
396	# <a href=...>Lateline</a>
397	# </span>
398	# <span class="itemTime">12:20pm</span>
399	# <span class="itemDesc">Current affairs program bla bla bla</span>
400	# </li>
401	# <li style="...">
402	# ...
403	# </li>
404	# </div>
405	#
406	# each <div id="epgAbcN" class="epgRowWrap"> is a channel
407	# each <li> is a programme
408	# each <span> is an attribute for the programme
409	#
410	# we need to extract $programme, $found_time, $progurl
411
412	my $divID=$chanID_to_divID{$chan_id};
413	if (!$divID) {
414	&log("failed to convert channel ID $chan_id to a <div> identifier");
415	die("failed to convert channel ID $chan_id to a <div> identifier");
416	}
417
418	for ($tree->look_down('_tag' => 'div', 'id' => $divID, 'class' => 'epgRowWrap')) {
419	# we have a channel
420	foreach my $tree_li ($_->look_down('_tag' => 'li')) {
421	# we have a programme
422	my %proghash;
423	my $programme;
424	my $progurl;
425	# scan through all the <span> entries looking for our data
426	foreach my $tree_span ($tree_li->look_down('_tag' => 'span')) {
427	# we have a programme attribute
428	my $tree_span_class = $tree_span->attr('class');
429	next if !defined $tree_span_class;
430	my $tree_span_value=$tree_span->as_text;
431	if ($tree_span_class eq "itemTitle") {
432	# have to get the link within the tag
433	# if it doesn't exist, must be "... Programs start at"
434	my $prog_a=$tree_span->look_down('_tag' => 'a');
435	next unless defined($prog_a);
436	$programme=$tree_span_value;
437	$progurl = $prog_a->attr('href');
438	}
439	$proghash{$tree_span_class}=$tree_span_value;
440	}
441	next unless $programme;
442
443	# we should now have all values we need
444	if ($proghash{itemTime} =~ /^(\d+):(\d+)(.)m/) {
445	$timeattr[2] = $1; # hour
446	$timeattr[1] = $2; # min
447
448	if ($3 eq "p") {
449	# pm
450	$timeattr[2] += 12 if ($timeattr[2] != 12);
451	$seen_pm = 1;
452	}
453	my $found_time = mktime(@timeattr);
454
455	# handle programmes that are after midnight
456	if (($seen_pm) && ($3 eq "a")) {
457	if ($timeattr[2] == 12) {
458	$found_time += (126060); # 12:xx am
459	} else {
460	$found_time += (246060);
461	}
462	}
463
464	if ($progurl =~ /^\/tv\/guide\//) {
465	printf "day %d time '%s' (%s) prog: %s url: %s\n",
466	#$daynum,$tree_row->as_text(),POSIX::strftime("%Y%m%d%H%M", localtime($found_time)),
467	$programme,$progurl if ($debug && $debug > 1);
468
469	$unprocessed_progname[$unprocessed_programmes] = $programme;
470	$unprocessed_starttime[$unprocessed_programmes] = $found_time;
471	$unprocessed_day[$unprocessed_programmes] = $daynum;
472	$unprocessed_url[$unprocessed_programmes] = "http://www.abc.net.au".$progurl;
473	$unprocessed_programmes++;
474	$seen_programmes++;
475	} else {
476	printf "ignoring prog %s because url %s is not a detail page\n",
477	$programme,$progurl if $debug;
478	}
479	}
480	}
481	}
482
483	$tree->delete;
484
485	if ($seen_programmes > 0) {
486	$stats{abc_daily_pages}++;
487
488	my $cache_key = sprintf "%d,%d,%s,%s", $currtime, 1, $chan_id, "Station Close";
489	if (defined $data_cache->{$cache_key}) {
490	# get station-close time from the previously-fetched "weekly programme guide"
491	&log(" Inserting 'Station Close'");
492
493	$unprocessed_progname[$unprocessed_programmes] = "Station Close";
494	$unprocessed_starttime[$unprocessed_programmes] = $data_cache->{$cache_key};
495	$unprocessed_day[$unprocessed_programmes] = $daynum;
496	$unprocessed_url[$unprocessed_programmes] = "";
497	$unprocessed_programmes++;
498	} elsif ($days_left == 0) {
499	# always dropped but fixes stop time for last program
500	&log(" Inserting 'Station Close Guess' for last day") if $debug;
501
502	$unprocessed_progname[$unprocessed_programmes] = "Station Close Guess";
503	$unprocessed_starttime[$unprocessed_programmes] = $currtime + (246060);
504	$unprocessed_day[$unprocessed_programmes] = $daynum;
505	$unprocessed_url[$unprocessed_programmes] = "";
506	$unprocessed_programmes++;
507	}
508	} else {
509	# if we were trying to fetch more than 7 days, stop on first day with no programmes
510	if ($daynum > 7) {
511	&log("failed to fetch $url, assuming we only have $daynum days..");
512	$days_left = 0;
513	next DAYS;
514	}
515	}
516	}
517
518	# have 'n' days of this channel unprocessed - process it!
519	&log((sprintf "Have summary data for %s, now fetching detail pages for up to %d programmes..",$chan_id,$unprocessed_programmes-2));
520
521	# only return programmes for requested days
522	@timeattr = localtime($script_start_time); # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
523	$timeattr[0] = 0; # zero seconds
524	$timeattr[1] = 0; # zero mintues
525	$timeattr[2] = 0; # zero hours
526	$timeattr[3] += $opt_offset; # day
527	my $start_day = mktime(@timeattr);
528	$timeattr[3] += $opt_days - $opt_offset; # day
529	my $stop_day = mktime(@timeattr);
530	$timeattr[3] -= $opt_days; # day
531
532	for (my $i = 0; $i < ($unprocessed_programmes-1); $i++) {
533	# if we've found a programme which is between days, evaluate whether we can
534	# use it or need to drop it (because we won't necessarily have a 'stop' time
535	if (($unprocessed_day[$i] != $unprocessed_day[($i+1)]) &&
536	($unprocessed_progname[$i] ne "Station Close")) {
537	# we'd normally throw away last programme from each day - we can't use it
538	# as we don't have a 'stop' time for it.
539	# with some ABC programming, ABC no longer indicate any station close
540	# on some days - they're broadcasting 24 hours.
541	# only throw away the last programme if:
542	# 1. it is seen to run longer than 3.5 hours
543	# 2. it is a different title
544	if ((($unprocessed_starttime[$i+1] - $unprocessed_starttime[$i]) > (3.5*3600)) &&
545	($unprocessed_progname[$i] ne $unprocessed_progname[$i+1])) {
546
547	# splice this in instead of dropping programs
548	splice @unprocessed_progname, $i+1, 0, ("Station Close Guess");
549	$timeattr[1] = 30; # minutes
550	$timeattr[2] = 4; # hours 4:30am
551	$timeattr[3] += $unprocessed_day[$i]; # day
552	splice @unprocessed_starttime, $i+1, 0, (mktime(@timeattr));
553	$timeattr[3] -= $unprocessed_day[$i]; # day
554	splice @unprocessed_day, $i+1, 0, ($unprocessed_day[$i+1]);
555	splice @unprocessed_url, $i+1, 0, ("");
556	$unprocessed_programmes++;
557
558	printf " Inserting after '%s' (%s), '%s' (%s) followed by '%s' (%s)\n",
559	$unprocessed_progname[$i],
560	POSIX::strftime("%Y%m%d%H%M", localtime($unprocessed_starttime[$i])),
561	$unprocessed_progname[$i+1],
562	POSIX::strftime("%Y%m%d%H%M", localtime($unprocessed_starttime[$i+1])),
563	$unprocessed_progname[$i+2],
564	POSIX::strftime("%Y%m%d%H%M", localtime($unprocessed_starttime[$i+2]));
565	}
566	}
567
568	# if we are micro-gap fetching, only include programmes which match our micro gaps
569	if ($opt_gaps_file ne "") {
570	my $found_gap_match = 0;
571	for (my $g_num = 0; $g_num <= $#gap_s; $g_num++) {
572	$found_gap_match++
573	if ((($gap_s[$g_num] >= $unprocessed_starttime[$i]) &&
574	($gap_s[$g_num] <= $unprocessed_starttime[$i+1])) \|\|
575	(($gap_e[$g_num] >= $unprocessed_starttime[$i]) &&
576	($gap_e[$g_num] <= $unprocessed_starttime[$i+1])) \|\|
577	(($gap_s[$g_num] <= $unprocessed_starttime[$i]) &&
578	($gap_e[$g_num] >= $unprocessed_starttime[$i+1])));
579	}
580	next if (!$found_gap_match);
581
582	$stats{programme_gaps_used}++;
583	printf "gap-fetching: including prog '%s', start %d, end %d\n", $unprocessed_progname[$i],
584	$unprocessed_starttime[$i], $unprocessed_starttime[$i+1] if $debug;
585	} elsif ($unprocessed_starttime[$i+1] <= $start_day or
586	$unprocessed_starttime[$i] >= $stop_day) {
587	printf "skipping program because before or after requested days, '%s', start %d, end %d\n",
588	$unprocessed_progname[$i], $unprocessed_starttime[$i], $unprocessed_starttime[$i+1] if $debug;
589	next;
590	}
591
592	$stats{programmes}++;
593	my $prog;
594
595	my $cache_key = sprintf "%d,%d,%s,%s", $unprocessed_starttime[$i], $unprocessed_starttime[$i+1], $xmlid, $unprocessed_progname[$i];
596
597	# The ABC guide normalizes everything to 5 minutes. This means that very
598	# short shows have the same start and stop time (a duration of 0 seconds),
599	# which Shepherd's reconciler will ignore as bad data. We will therefore
600	# enforce a minimum duration of 2 minutes.
601	#
602	if ($unprocessed_starttime[$i+1] <= $unprocessed_starttime[$i])
603	{
604	$unprocessed_starttime[$i+1] += (2*60);
605	}
606
607	$prog->{'channel'} = $xmlid;
608	$prog->{'start'} = POSIX::strftime("%Y%m%d%H%M", localtime($unprocessed_starttime[$i]));
609	$prog->{'stop'} = POSIX::strftime("%Y%m%d%H%M", localtime($unprocessed_starttime[$i+1]));
610	$prog->{'title'} = [[ $unprocessed_progname[$i], $lang ]];
611
612	if (defined $data_cache->{$cache_key}) {
613	$stats{used_cached_data}++;
614	} else {
615	if ((!$opt_cheap) && ($unprocessed_url[$i] ne "")) {
616	&get_one_abc_event($cache_key, $unprocessed_url[$i], $unprocessed_progname[$i]);
617
618	if (($stats{portal_detail_pages} % 25) == 1) {
619	&log((sprintf " .. at %s detail page %d of %d (used %d cached entries)",
620	$xmlid, ($i+1), $unprocessed_programmes-2,
621	(defined $stats{used_cached_data} ? $stats{used_cached_data} : 0)));
622
623	if (!$opt_fast) {
624	# slow down ..
625	my $waittime = 3 + int(rand(10));
626	sleep($waittime);
627	$stats{slept_for} += $waittime;
628	}
629	}
630	}
631	}
632
633	if (defined $data_cache->{$cache_key}) {
634	$prog->{'sub-title'} = [[ $data_cache->{$cache_key}->{subtitle}, $lang ]]
635	if $data_cache->{$cache_key}->{subtitle};
636	$prog->{'desc'} = [[ $data_cache->{$cache_key}->{desc}, $lang ]]
637	if $data_cache->{$cache_key}->{desc};
638	$prog->{'category'} = [[ $data_cache->{$cache_key}->{genre}, $lang ]]
639	if $data_cache->{$cache_key}->{genre};
640	$prog->{'previously-shown'} = { } if (defined $data_cache->{$cache_key}->{repeat});
641	$prog->{'subtitles'} = [ { 'type' => 'teletext' } ] if (defined $data_cache->{$cache_key}->{cc});
642	$prog->{'rating'} = [ [ $data_cache->{$cache_key}->{rating}, 'ABA', undef] ]
643	if (defined $data_cache->{$cache_key}->{rating});
644
645	if (defined $data_cache->{$cache_key}->{hdtv}) {
646	$prog->{video}->{aspect} = "16:9"; # widescreen
647	$prog->{video}->{quality} = "HDTV" if ($opt_no_hdtv_flag == 0);
648	}
649	}
650
651	Shepherd::Common::cleanup($prog);
652	$writer->write_programme($prog);
653	}
654	}
655
656	######################################################################################################
657
658	sub get_one_abc_event
659	{
660	my ($cache_key, $url, $orig_title) = @_;
661
662	if ($stats{failed_to_fetch_portal_detail_page} >= 3 or $stats{failed_to_parse_portal_detail_page} >= 9)
663	{
664	&log("skipping detail page because too many failures.");
665	return;
666	}
667
668	my $seen_programme = 0;
669	my $data = Shepherd::Common::get_url(url => $url);
670	my $tree = HTML::TreeBuilder->new_from_content($data) if ($data);
671	if (!defined $tree) {
672	&log("failed to fetch $url; skipping");
673	$stats{failed_to_fetch_portal_detail_page}++;
674	return;
675	}
676	Shepherd::Common::log("get_one_abc_event ".$url) if ($debug);
677
678	# Parse the page to get $full_title, $prog_genre_text, $prog_desc
679	# and:
680	# $data_cache->{$cache_key}->{repeat} = 1;
681	# $data_cache->{$cache_key}->{cc} = 1;
682	# $data_cache->{$cache_key}->{hdtv} = 1;
683	# $data_cache->{$cache_key}->{rating} = $1;
684	# Alas, nothing seems to be listing HDTV.
685	#
686	# page is of this form:
687	#
688	# <div id="maincontent">
689	# <div id="prgTop">
690	# <h1>Name of Programme - Subtitle</h1>
691	# <div id="prgSubDetails">
692	# <em id="prgGenre">Children's</em>
693	# <em id="prgRepeat">Repeat</em>
694	# <em id="prgCc">CC</em>
695	# <em id="prgRating">PG</em>
696	# <em id="prgDuration">15 mins</em>
697	# </div>
698	# </div>
699	# <div id="prgContent">
700	# <h3 id="aboutPrg>About the Program</h3>
701	# <p>This program is about bla bla, bla bla.</p>
702	# </div>
703	# <div id="sideInfo">
704	# <h4>Last Broadcast</h4>
705	# <p>8:00am Thu, August 7 on ABC3</p>
706	# </div>
707	# </div>
708
709	if (my $inner_tree = $tree->look_down('_tag' => 'div', 'id' => 'prgTop')) {
710	my $event_title = undef, my $event_subtitle = undef, my $event_description = undef, my $event_genre = undef;
711	my $event_repeat = undef, my $event_cc = undef, my $event_rating = undef;
712
713	if (my $prog_h1 = $inner_tree->look_down('_tag' => 'h1')) {
714	my $full_title = $prog_h1->as_HTML();
715	$full_title =~ s/(^<h1>\|<\/h1>$)//g;
716	chomp($full_title);
717	$full_title =~ s/\n$//s; # chomp doesn't seem to work
718	# TODO: check title shown here against the one from
719	# the overview, so as to avoid splitting on prognames
720	# with hiphens in them
721	($event_title,$event_subtitle) = split(/ - /,$full_title);
722
723	# If the title on this page matches the title
724	# on the parent page, we assume that the subtitle
725	# heuristics are correct (not a hiphenated title).
726	# Alas the event title sometimes contains url-escaped
727	# characters (such as &#39 for single quote) and
728	# so it will sometimes differ. We need to unescape
729	# before comparing.
730	if (1 or $event_title eq $orig_title) {
731
732	$event_title =~ s/(<[a-zA-Z0-9]+\>)//g; # remove html tags
733	$event_title =~ s/(^\n\|\n$)//g; # strip trailing/leading blank lines
734	Shepherd::Common::log(" - decoded title '".$event_title."' (cf '".$orig_title."')") if ($debug);
735
736	if ($event_subtitle) {
737	$event_subtitle =~ s/(<[\/a-zA-Z0-9]+\>)//g; # remove html tags
738	$event_subtitle =~ s/(^\n+\|\n+$)//g; # strip trailing/leading blank lines
739	$event_subtitle =~ s/(^\s+\|\s+$)//g; # strip trailing/leading blanks
740	if ($event_subtitle) {
741	$data_cache->{$cache_key}->{subtitle} = $event_subtitle;
742	Shepherd::Common::log(" - decoded subtitle '".$event_subtitle."'") if ($debug);
743	}
744	}
745	} elsif (length $event_subtitle) {
746	&log("confused by hiphen in title: '$full_title'; event title '$event_title' is not the same as original title '$orig_title'");
747	}
748	}
749
750	if (my $prog_sd = $inner_tree->look_down('_tag' => 'div', 'id' => 'prgSubDetails')) {
751	my %subdetails;
752	foreach my $em ($prog_sd->look_down('_tag' => 'em')) {
753	$subdetails{$em->attr('id')}=$em->as_text();
754	}
755	if (my $prog_genre_text = $subdetails{'prgGenre'}) {
756	$data_cache->{$cache_key}->{genre} = Shepherd::Common::translate_category($prog_genre_text);
757	Shepherd::Common::log(" - decoded genre '$prog_genre_text'") if ($debug);
758	}
759
760	if ($subdetails{'prgCc'}) {
761	$data_cache->{$cache_key}->{cc} = 1;
762	Shepherd::Common::log(" - decoded CC") if ($debug);
763	}
764
765	if ($subdetails{'prgHdtv'}) {
766	$data_cache->{$cache_key}->{hdtv} = 1;
767	Shepherd::Common::log(" - decoded HDTV") if ($debug);
768	}
769
770	if ($subdetails{'prgRepeat'}) {
771	$data_cache->{$cache_key}->{repeat} = 1;
772	Shepherd::Common::log(" - decoded Repeat") if ($debug);
773	}
774
775	# any remaining text should be rating
776	if (my $rating_text = $subdetails{'prgRating'}) {
777	$rating_text =~ s/[^\x20-\x7f]/ /g;
778	if ($rating_text =~ /^\s(\w+)\s$/) {
779	$data_cache->{$cache_key}->{rating} = $1;
780	Shepherd::Common::log(" - decoded Rating '$1'") if ($debug);
781	}
782	}
783	}
784
785	if (my $prog_desc = $tree->look_down('_tag' => 'div', 'class' => 'prgContent')) {
786	# gather description
787	foreach my $para ($prog_desc->look_down('_tag' => 'p')) {
788	$data_cache->{$cache_key}->{desc} .= $para->as_text() . "\n";
789	Shepherd::Common::log("added desc '".$para->as_text()."'") if ($debug);
790	}
791	}
792
793	if (defined $data_cache->{$cache_key}->{desc}) {
794	$data_cache->{$cache_key}->{desc} =~ s/(^\n\|\n$)//g; # strip trailing/leading blank lines
795	$data_cache->{$cache_key}->{desc} =~ s/(^\s+\|\s+$)//g; # strip trailing/leading spaces
796	delete $data_cache->{$cache_key}->{desc} if ($data_cache->{$cache_key}->{desc} eq "");
797	}
798
799	$seen_programme++;
800	$stats{added_cached_data}++;
801
802	&write_cache if (($opt_no_cache == 0) &&
803	(($stats{added_cached_data} % 30) == 0)); # incrementally write
804	}
805
806	$tree->delete;
807
808	if ($seen_programme == 0) {
809	printf "WARNING: failed to parse any programme data from '%s' - blocked/rate-limited/format-changed?\n",$url;
810	$stats{failed_to_parse_portal_detail_page}++;
811	return;
812	}
813	$stats{portal_detail_pages}++;
814	}
815
816	######################################################################################################
817
818	sub log
819	{
820	&Shepherd::Common::log(@_);
821	}
822
823	######################################################################################################
824
825	sub get_station_close
826	{
827	my ($chan_id,$url) = @_;
828	&log("Fetching weekly 'Station Close' data for $chan_id");
829	my $data = Shepherd::Common::get_url(url => $url);
830	my $tree = HTML::TreeBuilder->new_from_content($data) if ($data);
831
832	if (!defined $tree) {
833	&log("failed to fetch $url; skipping");
834	return;
835	}
836
837	# get date elements and optional time elements, in page order
838	my @elements = $tree->look_down(
839	sub {
840	# <h2 class="notfirst">Monday,ï¿œJulyï¿œ2</h2>
841	return 1 if (lc($_[0]->tag) eq 'h2' and $_[0]->as_text() !~ /Day Guide/i);
842
843	# <td>2:00am</td><td>... Programs start at 6.00am</td>
844	foreach my $item ($_[0]->content_list) {
845	next if ref $item;
846	return 1 if $item =~ /\.\.\.\s*programs start at /i;
847	}
848
849	return 0;
850	}
851	);
852
853	if (@elements == 0) {
854	&log("failed weekly parse because data layout changed.");
855	}
856
857	my %month_names = (january=>0,february=>1,march=>2,april=>3,may=>4,june=>5,
858	july=>6,august=>7,september=>8,october=>9,november=>10,december=>11);
859	my @timeattr = localtime($script_start_time); # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
860	$timeattr[0] = 0; # zero seconds
861
862	while (my $d = shift @elements) {
863	&log("date ".$d->as_text()) if $debug;
864
865	if (not $d->as_text() =~ /,.([a-z]+).([0-9]+)/i) {
866	&log("failed weekly parse because date layout changed.");
867	last;
868	}
869
870	$timeattr[1] = 30; # min
871	$timeattr[2] = 4; # hours 4:30am
872	$timeattr[3] = $2; # day
873	my $prev_month = $timeattr[4];
874	$timeattr[4] = $month_names{lc($1)}; # month
875	$timeattr[5]++ if ($prev_month == 11 and $timeattr[4] == 0); # year
876	my $cache_key = sprintf "%d,%d,%s,%s", mktime(@timeattr), 1, $chan_id, "Station Close";
877
878	# look ahead to see if a time element is next
879	if (@elements == 0 or lc($elements[0]->tag) eq 'h2') {
880	&log("station close time for $chan_id on $timeattr[3]/$timeattr[4] is not found, deleteing old time.") if $debug;
881	delete $data_cache->{$cache_key};
882	next;
883	}
884
885	my $t = shift @elements;
886	&log("time ".$t->left->as_text()." ".$t->as_text()) if $debug;
887
888	if ($t->left->as_text() =~ /^(\d+):(\d+)(.)m/) {
889	$timeattr[2] = $1; # hour
890	$timeattr[1] = $2; # min
891
892	if ($3 eq "p") {
893	# pm
894	$timeattr[2] += 12 if ($timeattr[2] != 12);
895	}
896	my $found_time = mktime(@timeattr);
897
898	if ($3 eq "a") {
899	# am - must be tomorrow
900	if ($timeattr[2] == 12) {
901	$found_time += (126060); # 12:xx am
902	} else {
903	$found_time += (246060);
904	}
905	}
906
907	$data_cache->{$cache_key} = $found_time;
908
909	printf "station close time for %s on $timeattr[3]/$timeattr[4] is %s\n",
910	$chan_id, POSIX::strftime("%Y%m%d%H%M", localtime($found_time)) if $debug;
911
912	# remove repeated time elements found on 2008/01/27
913	while (@elements > 0 and lc($elements[0]->tag) ne 'h2') {
914	&log("repeated station close time for $chan_id on $timeattr[3]/$timeattr[4] found, skipping it.");
915	&log("time ".$t->left->as_text()." ".$t->as_text()) if $debug;
916	shift @elements;
917	}
918
919	} else {
920	&log("skipping time because layout changed.");
921	}
922	}
923
924	$tree->delete;
925	}
926
927	######################################################################################################

Note: See TracBrowser for help on using the browser.

Context Navigation

root/trunk/grabbers/abc_website

Download in other formats: