Context Navigation

yahoo7web

Revision 1272, 22.1 kB (checked in by max, 2 years ago)
yahoo7web: Basic update to point to new source; still broken, needs rewrite to parse source correctly
Property svn:executable set to ``*

Line
1	#!/usr/bin/perl -w
2
3	# yahoo7portal au_tv guide grabber - runs from "Shepherd" master grabber
4	# * grabs data from the yahoo7portal (http://au.tv.yahoo.com/)
5	# * this does NOT use any config file - all settings are passed in from shepherd
6
7	use strict;
8
9	my $progname = "yahoo7web";
10	my $version = "0.99";
11	#
12	# This grabber currently broken! v0.99 points at new source,
13	# but still needs to be rewritten to parse new source format.
14	# The good news is that it doesn't need to request detail
15	# pages one at a time any more.
16	#
17	# June 2010.
18	#
19
20	use XMLTV;
21	use POSIX qw(strftime mktime);
22	use Getopt::Long;
23	use HTML::TreeBuilder;
24	use Data::Dumper;
25	use Shepherd::Common;
26
27	#
28	# global variables and settings
29	#
30
31	$\| = 1;
32	my $script_start_time = time;
33	my %stats;
34	my $channels, my $opt_channels, my $gaps;
35	my $data_cache;
36	my $writer;
37	my $prev_url;
38	my $d;
39	my $opt;
40
41	#
42	# parse command line
43	#
44
45	$opt->{days} = 7; # default
46	$opt->{outputfile} = "output.xmltv"; # default
47	$opt->{cache_file} = $progname.".storable.cache"; # default
48	$opt->{lang} = "en";
49	$opt->{region} = 94;
50
51	GetOptions(
52	'log-http' => \$opt->{log_http},
53	'region=i' => \$opt->{region},
54	'days=i' => \$opt->{days},
55	'offset=i' => \$opt->{offset},
56	'timezone=s' => \$opt->{timezone},
57	'channels_file=s' => \$opt->{channels_file},
58	'gaps_file=s' => \$opt->{gaps_file},
59	'output=s' => \$opt->{outputfile},
60	'cache-file=s' => \$opt->{cache_file},
61	'fast' => \$opt->{fast},
62	'no-cache' => \$opt->{no_cache},
63	'no-details' => \$opt->{no_details},
64	'debug+' => \$opt->{debug},
65	'warper' => \$opt->{warper},
66	'lang=s' => \$opt->{lang},
67	'obfuscate' => \$opt->{obfuscate},
68	'anonsocks=s' => \$opt->{anon_socks},
69	'help' => \$opt->{help},
70	'verbose' => \$opt->{help},
71	'version' => \$opt->{version},
72	'ready' => \$opt->{version},
73	'v' => \$opt->{help});
74
75	&help if ($opt->{help});
76
77	if ($opt->{version}) {
78	printf "%s %s\n",$progname,$version;
79	exit(0);
80	}
81
82	die "no channel file specified, see --help for instructions\n", if (!$opt->{channels_file});
83	$opt->{days} = 8 if ($opt->{days} > 8); # limit to a max of 8 days
84
85	#
86	# go go go!
87	#
88
89	&log(sprintf "going to %sgrab %d days%s of data into %s (%s%s%s%s%s)",
90	(defined $opt->{gaps_file} ? "micro-gap " : ""),
91	$opt->{days},
92	(defined $opt->{offset} ? " (skipping first $opt->{offset} days)" : ""),
93	$opt->{outputfile},
94	(defined $opt->{fast} ? "with haste" : "slowly"),
95	(defined $opt->{anon_socks} ? ", via multiple endpoints" : ""),
96	(defined $opt->{warper} ? ", anonymously" : ""),
97	(defined $opt->{no_details} ? ", without details" : ", with details"),
98	(defined $opt->{no_cache} ? ", without caching" : ", with caching"));
99
100	# set defaults
101	Shepherd::Common::set_default("debug", ($opt->{debug} ? 2 : 0));
102	Shepherd::Common::set_default("webwarper", 1) if (defined $opt->{warper});
103	Shepherd::Common::set_default("squid", 1) if (defined $opt->{obfuscate});
104	Shepherd::Common::set_default("referer", "last");
105	Shepherd::Common::setup_ua(cookie_jar => 1);
106
107	# read channels file
108	if (-r $opt->{channels_file}) {
109	local (@ARGV, $/) = ($opt->{channels_file});
110	no warnings 'all'; eval <>; die "$@" if $@;
111	} else {
112	die "WARNING: channels file $opt->{channels_file} could not be read\n";
113	}
114
115	# if just filling in microgaps, parse gaps
116	if (defined $opt->{gaps_file}) {
117	if (-r $opt->{gaps_file}) {
118	local (@ARGV, $/) = ($opt->{gaps_file});
119	no warnings 'all'; eval <>; die "$@" if $@;
120	} else {
121	die "WARNING: gaps_file $opt->{gaps_file} could not be read: $!\n";
122	}
123	}
124
125	&read_cache unless (defined $opt->{no_cache});
126
127	if (defined $opt->{anon_socks}) {
128	&log("configured to use Tor, testing that it works by connecting to www.google.com ...");
129	if (Shepherd::Common::setup_socks($opt->{anon_socks})) {
130	&log("success. Tor appears to be working!");
131	} else {
132	&log("ERROR: Could not connect to www.google.com via Tor, disabling Tor.");
133	&log(" DATA FETCHING WILL BE VERY SLOW.");
134	&log(" DISABLING DETAILS-FETCHING BECAUSE OF THIS - SIGNIFICANTLY LOWER DATA QUALITY!!");
135
136	$opt->{no_details} = 1;
137	delete $opt->{anon_socks};
138	$stats{fallback_to_non_tor}++;
139	}
140	}
141
142	&start_writing_xmltv;
143
144	&get_summary_pages;
145	&get_detailed_pages;
146
147	$writer->end();
148
149	&write_cache unless (defined $opt->{no_cache});
150
151	&print_stats;
152	exit(0);
153
154	##############################################################################
155	# help
156
157	sub help
158	{
159	print<<EOF
160	$progname $version
161
162	options are as follows:
163	--help show these help options
164	--days=N fetch 'n' days of data (default: $opt->{days})
165	--output=file send xml output to file (default: "$opt->{outputfile}")
166	--no-cache don't use a cache to optimize (reduce) number of web queries
167	--no-details don't fetch detailed descriptions (default: do)
168	--cache-file=file where to store cache (default "$opt->{cache_file}")
169	--fast don't run slow - get data as quick as you can - not recommended
170	--anonsocks=(ip:port) use SOCKS4A server at (ip):(port) (for Tor: recommended)
171
172	--debug increase debug level
173	--warper fetch data using WebWarper web anonymizer service
174	--obfuscate pretend to be a proxy servicing multiple clients
175	--lang=[s] set language of xmltv output data (default $opt->{lang})
176
177	--region=N set region for where to collect data from (default: $opt->{region})
178	--channels_file=file where to get channel data from
179	--gaps_file=file micro-fetch gaps only
180
181	EOF
182	;
183
184	exit(0);
185	}
186
187	##############################################################################
188	# populate cache
189
190	sub read_cache
191	{
192	my $store = Shepherd::Common::read_cache(\$opt->{cache_file});
193
194	if ($store) {
195	$data_cache->{progs} = $store->{data_cache} if (defined $store->{data_cache});
196
197	if (defined $store->{day_cache}) {
198	$data_cache->{day} = $store->{day_cache};
199
200	# age day cache on reading..
201	for my $url (keys %{($data_cache->{day})}) {
202	if ($data_cache->{day}->{$url}->{fetched} < (time-(4*3600))) {
203	delete $data_cache->{day}->{$url};
204	$stats{expired_url_from_cache}++;
205	}
206	}
207	}
208	}
209	}
210
211	##############################################################################
212	# write out updated cache
213
214	sub write_cache
215	{
216	# cleanup old prog entries from cache
217	if (defined $data_cache->{progs}) {
218	for my $cache_key (keys %{($data_cache->{progs})}) {
219	my ($starttime, @rest) = split(/:/,$cache_key);
220	if ($starttime < (time-86400)) {
221	delete $data_cache->{progs}->{$cache_key};
222	$stats{expired_from_cache}++;
223	}
224	}
225	}
226
227	my $store = { };
228	$store->{data_cache} = $data_cache->{progs} if (defined $data_cache->{progs});
229	$store->{day_cache} = $data_cache->{day} if (defined $data_cache->{day});
230	Shepherd::Common::write_cache($opt->{cache_file}, $store);
231	}
232
233	##############################################################################
234
235	sub log
236	{
237	my ($entry) = @_;
238	printf "%s\n",$entry;
239	}
240
241	##############################################################################
242
243	sub print_stats
244	{
245	printf "STATS: %s v%s completed in %d seconds",$progname, $version, time-$script_start_time;
246	foreach my $key (sort keys %stats) {
247	printf ", %d %s",$stats{$key},$key;
248	}
249	printf "\n";
250	}
251
252	##############################################################################
253
254	sub start_writing_xmltv
255	{
256	my %writer_args = ( encoding => 'ISO-8859-1' );
257	if ($opt->{outputfile}) {
258	my $fh = new IO::File(">$opt->{outputfile}") \|\| die "can't open $opt->{outputfile}: $!";
259	$writer_args{OUTPUT} = $fh;
260	}
261
262	$writer = new XMLTV::Writer(%writer_args);
263
264	$writer->start
265	( { 'source-info-name' => "$progname $version",
266	'generator-info-name' => "$progname $version"} );
267
268	for my $channel (sort keys %{$channels}) {
269	$writer->write_channel( {
270	'display-name' => [[ $channel, $opt->{lang} ]],
271	'id' => $channels->{$channel}
272	} );
273	}
274	}
275
276	##############################################################################
277
278	sub translate_category
279	{
280	my $genre = shift;
281	my %translation = (
282	'Sport' => 'sports',
283	'Soap Opera' => 'Soap',
284	'Science and Technology' => 'Science/Nature',
285	'Real Life' => 'Reality',
286	'Cartoon' => 'Animation',
287	'Family' => 'Children',
288	'Murder' => 'Crime' );
289
290	return $translation{$genre} if defined $translation{$genre};
291	return $genre;
292	}
293
294	##############################################################################
295
296	sub build_channel_quirks_map
297	{
298	# set up channel name exceptions list
299	my %chan_map;
300	# if ($opt->{region} == 90) {
301	# # VIC: Eastern Victoria
302	# push (@{($chan_map{"Prime"})},
303	# "Prime (Regional Victoria)",
304	# "Prime (Albury)");
305	# }
306
307
308	if ($opt->{region} == 95) {
309	# VIC: Mildura/Sunraysia
310	push (@{($chan_map{"TEN"})},
311	"TEN (VIC: Mildura/Sunraysia)",
312	"TEN (Mildura Digital)");
313	}
314
315	return %chan_map;
316	}
317
318	##############################################################################
319
320	sub get_summary_pages
321	{
322	my $starttime = time;
323	my $day_num = 0;
324	my $skip_days = 0;
325	my $prev_http_successful_requests = 0;
326	$stats{programmes} = 0;
327
328	my @timeattr = localtime($starttime); # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
329	$timeattr[0] = 0; # zero sec
330	$timeattr[1] = 0; # zero min
331	$timeattr[2] = 0; # zero hour (midnight)
332	my $starttime_midnight = mktime(@timeattr);
333
334	$skip_days = $opt->{offset} if (defined $opt->{offset});
335	while ($day_num < $opt->{days}) {
336	my $day_start = $starttime_midnight + (606024 * $day_num);
337	$day_num++;
338
339	# skip if --offset applies against this day
340	if ($skip_days > 0) {
341	$skip_days--;
342	next;
343	}
344
345	# within each day, fetch in groups of 3 hours
346	for (my $hr = 0; $hr < 23; $hr += 3) {
347	my $currtime = $day_start + ($hr * 60 * 60);
348	next if (($currtime + (3 * 60 * 60)) < $starttime); # no point fetching the past
349
350	# if we are fetching microgaps, skip this summary page if we aren't
351	# interested in anything from it anyway
352	next if ((defined $opt->{gaps_file}) && (!window_is_within_microgap($currtime,$currtime+(60603))));
353
354	# Format change!
355	#
356	# Datasource now offers lots of data with a single download, but in new format.
357	#
358	# my $url = sprintf "http://au.tv.yahoo.com/tv-guide/?hour=%s&min=%s&date=%s&mon=%s&year=%s&tvrg=%s&next=%s",
359	# POSIX::strftime("%H",localtime($starttime_midnight)),
360	# POSIX::strftime("%M",localtime($starttime_midnight)),
361	# POSIX::strftime("%d",localtime($starttime_midnight)),
362	# POSIX::strftime("%m",localtime($starttime_midnight)),
363	# POSIX::strftime("%Y",localtime($starttime_midnight)),
364	# $opt->{region}, $currtime;
365
366	my $url = sprintf 'http://au.tv.yahoo.com/tv-guide/%d/%d/%d/%d/',
367	$opt->{'region'},
368	0, # Free-to-air
369	$day_num - 1,
370	$hr;
371
372	&log("fetching day $day_num summary page hour $hr ($url)");
373	&parse_summary_page($url, $day_num, $day_start);
374	}
375
376	if ((defined $stats{http_successful_requests}) && ($stats{http_successful_requests} > $prev_http_successful_requests)) {
377	$prev_http_successful_requests = $stats{http_successful_requests};
378	my $wait_for = 5 + int(rand(5));
379	$stats{slept_for} += $wait_for;
380	sleep($wait_for);
381	}
382	}
383	}
384
385	##############################################################################
386
387	sub parse_summary_page
388	{
389	my ($url, $day_num, $day_start) = @_;
390	my %chan_map = &build_channel_quirks_map;
391	my $data;
392	my $first_time = 1;
393
394	if ((defined $data_cache->{day}->{$url}) &&
395	(defined $data_cache->{day}->{$url}->{data})) {
396	$data = $data_cache->{day}->{$url}->{data};
397	$stats{used_cached_day_page}++;
398	} else {
399	$data = Shepherd::Common::get_url(url => $url, retries => 4);
400	if (!$data) {
401	if (!$stats{summary_pages_with_progs}) {
402	&log("Aborting: couldn't fetch first summary page.");
403	exit 10;
404	}
405	return;
406	}
407	$data_cache->{day}->{$url}->{fetched} = time;
408	$data_cache->{day}->{$url}->{data} = $data;
409
410	my $wait_for = 2;
411	$stats{slept_for} += $wait_for;
412	sleep($wait_for);
413	}
414
415	my $tree = HTML::TreeBuilder->new_from_content($data);
416	if (!$tree) {
417	&log("Format change? No valid HTML in $url");
418	exit 20 unless ($stats{summary_pages_with_progs})
419	}
420
421	my $tree_table = $tree->look_down('_tag' => 'div', 'class' => 'bd');
422	if (!$tree_table) {
423	&log("Format change? No TV table in $url?");
424	exit 20 unless ($stats{summary_pages_with_progs});
425	}
426
427	my $progs_in_table = 0;
428
429	for my $tree_tr ($tree_table->look_down('_tag' => 'li', 'class' => 'row channel')) {
430	# get channel
431	my $this_chan = "";
432	if (my $channel_td = $tree_tr->look_down('_tag' => 'h3')) {
433	$this_chan = $channel_td->as_text();
434	}
435
436	if ($this_chan eq "") {
437	&log("ignoring blank channel in $url") if (defined $opt->{debug});
438	$stats{blank_channels_ignored}++;
439	next;
440	}
441
442	if (defined $chan_map{$this_chan}) {
443	my $new_channame = splice(@{($chan_map{$this_chan})},0,1);
444	if (not $new_channame) {
445	&log("new unmapped channel for '$this_chan'");
446	} else {
447	&log("substituted channel name '$new_channame' for '$this_chan'") if (defined $opt->{debug});
448	$stats{substituted_channels}++;
449	$this_chan = $new_channame;
450	}
451	}
452
453	if (!defined $channels->{$this_chan}) {
454	&log("skipping unlisted channel '$this_chan'") if (!defined $d->{skipped_channels}->{$this_chan});
455	$d->{skipped_channels}->{$this_chan} = 1 if (!defined $opt->{debug});
456	$stats{skipped_channels}++;
457	next;
458	}
459
460	for my $tree_td ($tree_tr->look_down('_tag' => 'li', 'class' => 'item')) {
461	if (my $listing_div = $tree_td->look_down('_tag' => 'div')) {
462	next if ($listing_div->attr('class') !~ /^lt-listing-wrapper/i);
463
464	my @listing_links = $listing_div->look_down('_tag' => 'a', 'class' => 'listing-link');
465	my @listing_data = $listing_div->look_down('_tag' => 'strong');
466
467	for (my $i=0; $i <= $#listing_links; $i++) {
468	my $prog;
469	$prog->{channel} = $channels->{$this_chan};
470
471	if ($listing_links[$i]->attr('rel') =~ /^(\d+)-(\d+)-(\d+)$/) {
472	$prog->{event_id} = $3;
473	}
474	$prog->{title} = [[ $listing_links[$i]->as_text(), $opt->{lang} ]];
475
476	my $listing_text = $listing_data[$i]->as_text();
477	if ($listing_text =~ /^(.*)$(\d+)$(\d+):(\d+)(.)m - (\d+):(\d+)(.)m$/i) {
478	my ($rating_text, $prog_length, $start_sec, $stop_sec) = ($1, $2, parse_time($3, $4, $5), parse_time($6, $7, $8));
479	if ($stop_sec < $start_sec) { # program wrap around midnight
480	if ($first_time) {
481	$start_sec -= (606024);
482	} else {
483	$stop_sec += (606024);
484	}
485	}
486	$first_time = 0;
487	$prog->{rating} = [[ $rating_text, 'ABA', undef ]] if ((defined $rating_text) && ($rating_text ne ""));
488	$prog->{length} = ($prog_length * 60) if ((defined $prog_length) && ($prog_length > 0));
489	$prog->{starttime} = $day_start + $start_sec;
490	$prog->{stoptime} = $day_start + $stop_sec;
491	} else {
492	&log("malformed listing_text '$listing_text' for prog '".$listing_links[$i]->as_text()."'; ignored.");
493	$stats{malformed_listing}++;
494	next;
495	}
496
497	$progs_in_table++;
498
499	# if we are fetching microgaps, skip if this isn't in a micro-gap.
500	if (defined $opt->{gaps_file}) {
501	next if (!window_is_within_microgap($prog->{starttime},$prog->{stoptime},$this_chan));
502	$stats{gaps_included}++;
503	}
504
505	# include programme
506	&log("found prog: '".$prog->{title}->[0]->[0]."', channel ".$prog->{channel}.
507	" start ".$prog->{starttime}." stop ".$prog->{stoptime}) if (defined $opt->{debug});
508
509	my $cache_key = sprintf "%d:%d:%s:%s", $prog->{starttime}, $prog->{stoptime}, $prog->{channel}, $prog->{title}->[0]->[0];
510	if (!defined $d->{progs}->{$cache_key}) {
511	$d->{progs}->{$cache_key} = $prog;
512	$stats{programmes}++;
513	}
514	}
515	}
516	}
517	}
518
519	$tree->delete;
520
521	$stats{summary_pages_with_progs}++ if ($progs_in_table > 0);
522
523	&log("WARNING: Data may be bad. Only $progs_in_table programmes seen in $url") if ($progs_in_table * scalar(keys %$channels) < 4);
524	}
525
526	##############################################################################
527	# loop through our progs, fetching details where we don't have a pre-cached
528	# entry for them.
529	# write out XMLTV
530
531	sub get_detailed_pages
532	{
533	&log("fetching details for up to ".$stats{programmes}." programmes ...") if (!defined $opt->{no_details});
534
535	my $prog_count = 0;
536	my $added_to_cache = 0;
537	$stats{used_existing_cache_entry} = 0;
538	$stats{added_to_cache} = 0;
539	my $abc2_eariest_start = "300000000000";
540
541	foreach my $cache_key (sort keys %{($d->{progs})}) {
542	my $prog = $d->{progs}->{$cache_key};
543	$prog_count++;
544
545	if ((!defined $data_cache->{progs}->{$cache_key}) &&
546	(!defined $opt->{no_details}) &&
547	(defined $prog->{event_id}) &&
548	($prog->{title}->[0]->[0] ne "Station Close")) {
549	&fetch_one_prog($cache_key, $prog->{event_id});
550	&write_cache if ((($stats{added_to_cache} % 15) == 0) && (!defined $opt->{no_cache}));
551	} elsif (!defined $opt->{no_details}) {
552	$stats{used_existing_cache_entry}++;
553	}
554
555	if ((($prog_count % 25) == 0) && (!defined $opt->{no_details})) {
556	&log(" ... at ".$prog_count." of ".$stats{programmes}." programmes (used ".$stats{used_existing_cache_entry}." from cache)");
557	}
558
559	# if we got additional details from the cache, add them now
560	if (defined $data_cache->{progs}->{$cache_key}) {
561	foreach my $key (keys %{($data_cache->{progs}->{$cache_key})}) {
562	$prog->{$key} = $data_cache->{progs}->{$cache_key}->{$key};
563	}
564	}
565
566	# convert epoch starttime into XMLTV starttime
567	$prog->{start} = POSIX::strftime("%Y%m%d%H%M", localtime($prog->{starttime}));
568	delete $prog->{starttime};
569
570	# convert epoch stoptime into XMLTV stoptime
571	$prog->{stop} = POSIX::strftime("%Y%m%d%H%M", localtime($prog->{stoptime}));
572	delete $prog->{stoptime};
573
574	delete $prog->{event_id};
575	Shepherd::Common::cleanup($prog);
576
577	printf "DEBUG: programme xmltv: ".Dumper($prog) if ((defined $opt->{debug}) && ($opt->{debug} > 1));
578	$writer->write_programme($prog);
579
580	$abc2_eariest_start = $prog->{start}
581	if (defined $channels->{ABC2} &&
582	$prog->{channel} eq $channels->{ABC2} && $abc2_eariest_start > $prog->{start});
583	}
584
585	# check if abc2 has a gap on the first day when the station is closed
586	if ($abc2_eariest_start != "300000000000" && (!defined $opt->{offset}) && defined $channels->{ABC2} &&
587	$abc2_eariest_start > POSIX::strftime("%Y%m%d%H%M", localtime($script_start_time))) {
588
589	# create 7am today
590	my @timeattr = localtime($script_start_time); # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
591	$timeattr[0] = 0; # zero seconds
592	$timeattr[1] = 0; # min
593	$timeattr[2] = 7; # hours 7am
594	my $time7am = mktime(@timeattr);
595	my $xmltime7am = POSIX::strftime("%Y%m%d%H%M", localtime($time7am));
596
597	if (($script_start_time < $time7am) && ($abc2_eariest_start == $xmltime7am) &&
598	(!defined $opt->{gaps_file} \|\|
599	window_is_within_microgap($script_start_time, $time7am, "ABC2")))
600	{
601	my $show;
602	$show->{'channel'} = $channels->{ABC2};
603	$show->{'title'} = [[ "Station Close Guess", $opt->{lang} ]];
604	$show->{'start'} = POSIX::strftime("%Y%m%d%H%M", localtime($script_start_time));
605	$show->{'stop'} = $abc2_eariest_start;
606
607	Shepherd::Common::cleanup($show);
608	$writer->write_programme($show);
609	}
610	}
611	}
612
613	##############################################################################
614
615	sub fetch_one_prog
616	{
617	my ($cache_key, $event_id) = @_;
618	&log("fetching detail page for $cache_key with event_id $event_id") if (defined $opt->{debug});
619
620	my $url = "http://au.tv.yahoo.com/tv-guide/broker.html?event_id=".$event_id;
621	my $data = Shepherd::Common::get_url(url => $url, retries => 4);
622
623	if ((!$data) \|\| ($data !~ /^\{.*\}$/)) {
624	$stats{bad_details_page}++;
625	return;
626	}
627
628	$stats{added_to_cache}++;
629
630	if (($stats{added_to_cache} % 35) == 0) {
631	my $wait_for = 12 + int(rand(5));
632	$stats{slept_for} += $wait_for;
633	sleep $wait_for;
634	}
635
636	my @genre;
637
638	$data =~ s/(^\{\|\}$)//g; # strip leading/trailing { and }
639	foreach my $field_item (split(/,"/,$data)) {
640	if ($field_item =~ /^([A-Za-z0-9\_\"]+):(.*)/) {
641	my ($f, $v) = ($1, $2);
642	$f =~ s/(^\"\|\"$)//g; # strip leading/trailing quotes from field if present
643	$v =~ s/(^\"\|\"$)//g; # strip leading/trailing quotes from value if present
644	next if ($v eq "");
645
646	if ($f eq "title") {
647	; # nothing
648	} elsif ($f eq "subtitle") {
649	$data_cache->{progs}->{$cache_key}->{'sub-title'} = [[ $v, $opt->{lang} ]];
650	} elsif ($f eq "description") {
651	$data_cache->{progs}->{$cache_key}->{desc} = [[ $v, $opt->{lang} ]];
652	} elsif ($f eq "genre") {
653	push(@genre, translate_category($v));
654	} elsif ($f eq "captions") {
655	$data_cache->{progs}->{$cache_key}->{subtitles} = [ { 'type' => 'teletext' } ] if ($v eq "true");
656	} elsif ($f eq "repeat" and $v and $v eq "true") {
657	$data_cache->{progs}->{$cache_key}->{'previously-shown'} = { };
658	} elsif ($f eq "start_date") {
659	; # nothing
660	} elsif ($f eq "end_date") {
661	; # nothing
662	} elsif ($f eq "rating") {
663	; # nothing
664	} elsif ($f eq "channel") {
665	; # nothing
666	} elsif ($f eq "hotpick") {
667	; # nothing
668	} elsif ($f eq "venue_url") {
669	; # nothing
670	} elsif ($f eq "url") {
671	; # nothing
672	} elsif ($f eq "alt_url") {
673	; # nothing
674	} elsif ($f eq "alt_text") {
675	; # nothing
676	} elsif ($f eq "img") {
677	; # nothing
678	} else {
679	&log("unknown field '$f' in $url") if (!defined $d->{unknown_fields}->{$f});
680	$d->{unknown_fields}->{$f} = 1;
681	}
682
683	$data_cache->{progs}->{$cache_key}->{category} = [[ @genre ]] if ($#genre != -1);
684
685	} else {
686	&log("unknown field format '$field_item' in details. Has the format changed?");
687	$stats{unknown_details_field_format}++;
688	}
689	}
690
691	printf "DEBUG: cached entries for '$cache_key': ".Dumper($data_cache->{progs}->{$cache_key})
692	if (defined $opt->{debug});
693	}
694
695	##############################################################################
696
697	sub parse_time
698	{
699	my ($hr, $min, $ampm) = @_;
700
701	$hr = 0 if ($hr == 12);
702	$hr += 12 if ($ampm =~ /p/i);
703
704	return(($hr6060)+($min*60));
705	}
706
707	##############################################################################
708
709	sub window_is_within_microgap
710	{
711	my ($start, $stop, $channel) = @_;
712
713	return window_channel_is_within_microgap($start, $stop, $channel) if (defined $channel);
714
715	foreach my $ch (keys %{$channels}) {
716	return 1 if window_channel_is_within_microgap($start, $stop, $ch);
717	}
718	return 0;
719	}
720
721	sub window_channel_is_within_microgap
722	{
723	my ($start, $stop, $channel) = @_;
724
725	if (defined $gaps->{$channel}) {
726	foreach my $g (@{($gaps->{$channel})}) {
727	my ($s, $e) = split(/-/,$g);
728	return 1 if
729	((($s >= $start) && ($s <= $stop)) \|\|
730	(($e >= $start) && ($e <= $stop)) \|\|
731	(($s <= $start) && ($e >= $stop)));
732	}
733	}
734	$stats{gaps_skipped}++;
735	return 0;
736	}
737
738	##############################################################################

Note: See TracBrowser for help on using the browser.

Context Navigation

root/trunk/grabbers/yahoo7web

Download in other formats: