Context Navigation

tvdb_augment_data

Revision 1109, 33.8 kB (checked in by paul, 4 years ago)
tvdb_augment_data: add desc to series info when replacing
Property svn:executable set to ``*

Line
1	#!/usr/bin/perl -w
2
3	# tvdb (thetvdb.com / tv.com) XMLTV data augmenter <ltd@interlink.com.au>
4	#
5	# * to be used as a postprocessor for XMLTV data
6	# * uses data from thetvdb.com to augment TV guide data
7	# * this should only be used for non-commercial use.
8	# * can be used in conjunction with 'shepherd' XMLTV reconciler or standalone
9	# (pipe-through)
10	# * no configuration necessary
11	#
12	# thanks to Scott Zsori, Paul Taylor, Josh Walters and the other thetvdb.com
13	# folks for providing the data and an easy interface/schema to search for data!
14	#
15	# changelog:
16	# 0.01 31may07 ltd initial version
17	# 0.02 fill in actors/writers/directors, augment description if there is none
18
19	use strict;
20
21	my $progname = "tvdb_augment_data";
22	my $version = "0.14";
23	my $mirrorlist_url = 'http://thetvdb.com/interfaces/GetMirrors.php';
24
25	use XMLTV;
26	use XML::DOM;
27	use Getopt::Long;
28	use HTML::TreeBuilder;
29	use Shepherd::Common;
30	use Data::Dumper;
31	use Encode;
32
33	#
34	# some initial cruft
35	#
36
37	my $script_start_time = time;
38	my %stats;
39	my $data_cache;
40	my $settings_override = { };
41	my $d;
42	my $max_failures = 5;
43	my $max_server_retries = 3;
44	my $first_parse_error = 1;
45	my $parser = new XML::DOM::Parser;
46
47	$\| = 1;
48
49	#
50	# parse command line
51	#
52
53	my $opt = { };
54	$opt->{output_file} = "output.xmltv";
55	$opt->{cache_file} = $progname.".storable.cache";
56	$opt->{lang} = "en";
57	$opt->{debug} = 0;
58	$opt->{min_duration} = 25; # 25 mins
59	$opt->{max_duration} = 140; # 2 hrs 20 mins
60	$opt->{skip_categories} = "Infotainment,Shopping,Business and Finance,Game Show,News,Parliament,Current Affairs,sports,Sport,Weather,Reality,live,Movies,Movie";
61	$opt->{cache_details_for} = 60; # cache series details for up to 2 months
62	$opt->{cache_title_for} = 90; # cache title lookups for 3 months
63
64	GetOptions(
65	'region=i' => \$opt->{region}, # ignored
66	'days=i' => \$opt->{days}, # ignored
67	'offset=i' => \$opt->{offset}, # ignored
68	'timezone=s' => \$opt->{timezone}, # ignored
69	'channels_file=s' => \$opt->{channels_file}, # ignored
70	'config-file=s' => \$opt->{configfile}, # ignored
71
72	'min_duration=i' => \$opt->{min_duration},
73	'max_duration=i' => \$opt->{max_duration},
74	'skip_categories=s' => \$opt->{skip_categories},
75	'cache_details_for=i' => \$opt->{cache_details_for},
76	'cache_title_for=i' => \$opt->{cache_title_for},
77	'long-info' => \$opt->{long_info},
78
79	'dump-cache' => \$opt->{dump_cache},
80
81	'output=s' => \$opt->{output_file},
82	'cache-file=s' => \$opt->{cache_file},
83	'fast' => \$opt->{fast},
84	'debug+' => \$opt->{debug},
85	'lang=s' => \$opt->{lang},
86	'help' => \$opt->{help},
87	'set=s' => \$opt->{set},
88	'verbose' => \$opt->{help},
89	'version' => \$opt->{version},
90	'ready' => \$opt->{ready},
91	'desc' => \$opt->{desc},
92	'v' => \$opt->{version});
93
94	&dump_cache if (defined $opt->{dump_cache});
95	&set_settings if (defined $opt->{set});
96
97	if ($opt->{version} \|\| $opt->{desc} \|\| $opt->{help} \|\| $opt->{ready} \|\|
98	$opt->{output_file} eq "" \|\| (scalar @ARGV == 0)) {
99	printf "%s v%s\n",$progname,$version;
100	printf "Augments XMLTV data with programme information from thetvdb.com\n" if $opt->{desc};
101	printf "$progname is ready for operation.\n" if ($opt->{ready});
102	printf "No --output file specified.\n" if ($opt->{output_file} eq "");
103	printf "No input XMLTV files specified.\n" if (scalar @ARGV == 0);
104
105	&help if ($opt->{help} \|\| $opt->{output_file} eq "" \|\| (scalar @ARGV == 0));
106	exit(0);
107	}
108
109	&set_settings if (defined $opt->{set});
110
111	# set defaults
112	Shepherd::Common::set_default("debug", (($opt->{debug} > 0) ? 2 : 0));
113	Shepherd::Common::set_default("stats" => \%stats);
114	Shepherd::Common::set_default("retry_delay" => "15-45");
115	Shepherd::Common::set_default("delay" => "0-4") unless (defined $opt->{fast});
116	Shepherd::Common::set_default('fake' => 0);
117
118	# go go go!
119
120	Shepherd::Common::log(sprintf "%s v%s started: %s%soutput %s\n",
121	$progname, $version,
122	($opt->{fast} ? "fast-override, " : ""),
123	($opt->{debug} ? "debug enabled, " : ""),
124	($opt->{output_file}));
125
126	&read_cache;
127
128	Shepherd::Common::log("Stage 1/5: reading input xmltv files...");
129	foreach my $file (@ARGV) {
130	&read_xmltv($file);
131	}
132
133	eval{ &perform_lookups };
134	if ($@) {
135	Shepherd::Common::log("WARNING: failed to perform lookups : $@");
136	#my $error = $@;
137	#$error =~ s/ /_/g;
138	#$stats{"failed_to_perform_lookups_$error"} = 1;
139	$stats{failed_to_perform_lookups}++;
140	} else {
141	$stats{failed_to_perform_lookups} = 0;
142	}
143	&write_xmltv;
144	&write_cache;
145	Shepherd::Common::print_stats($progname, $version, $script_start_time, %stats);
146	exit(0);
147
148	##############################################################################
149
150	sub help
151	{
152	print<<EOF
153	usage: $0 [options] {FILE(s)}
154
155	Supported options include:
156	--min_duration={min} ignore programs under {min} duration (default: $opt->{min_duration} min)
157	--max_duration={min} ignore programs over {min} duration (default: $opt->{max_duration} min)
158	--skip_categories={list} don't try to look up programmes in these categories (default: $opt->{skip_categories})
159
160	--long-info provide LOTS of program info (default: don't)
161
162	--cache_details_for={days} cache programme details for {days} (def: $opt->{cache_details_for} days)
163	--cache_title_for={days} cache URLs for {days} (def: $opt->{cache_title_for} days)
164
165	--lang={lang} set language to {lang} (default: $opt->{lang})
166	--output={file} send final XMLTV output to {file} (default: $opt->{output_file})
167	--debug enable debugging
168	--fast don't pause between requests to server
169
170	--cache-file={file} local file to use as our data cache (default: $opt->{cache_file})
171	--no-cache don't use local cache to reduce network load on server
172
173	--set=(setting):(value) save setting override: (value) 1=enable, 0=disable
174	dont_augment_desc:1/0 (don't / do)
175
176	--dump-cache (debugging) show cache contents
177
178	EOF
179	;
180	}
181
182	##############################################################################
183
184	sub dump_cache
185	{
186	&read_cache;
187	$Data::Dumper::Indent = 1;
188	print Dumper($data_cache);
189	exit(0);
190	}
191
192	##############################################################################
193
194	sub set_settings
195	{
196	&read_cache;
197	my ($setting, $val) = split(/:/,$opt->{set});
198
199	die "--set format is (setting):(value) where value is 0 for disable, 1 for enable.\n"
200	if ((!defined $val) \|\| (($val ne "0") && ($val ne "1")));
201
202	die "unknown '--set' parameter '$setting', see --help for details.\n"
203	if ($setting ne "dont_augment_desc");
204
205	$settings_override->{$setting} = $val;
206	printf "%s: override parameter %s: %s\n", $progname, $setting, ($val eq "0" ? "disabled" : "enabled");
207
208	&write_cache;
209	exit(0);
210	}
211
212	##############################################################################
213	# populate cache
214
215	sub read_cache
216	{
217	my $store = Shepherd::Common::read_cache(\$opt->{cache_file});
218
219	if ($store) {
220	$data_cache = $store->{data_cache};
221	$settings_override = $store->{settings_override};
222	$stats{failed_to_perform_lookups} = $store->{failed_to_perform_lookups};
223
224	foreach my $setting (keys %$settings_override) {
225	$opt->{$setting} = 1 if ($settings_override->{$setting} != 0);
226	}
227
228	#
229	# age our caches on startup
230	#
231
232	# age our programme cache on startup
233	foreach my $key (keys %{($data_cache->{prog})}) {
234	my $num_items = 0;
235	foreach my $key2 (keys %{($data_cache->{prog}->{$key})}) {
236	if ($data_cache->{prog}->{$key}->{$key2}->{expires} < $script_start_time) {
237	delete $data_cache->{prog}->{$key}->{$key2};
238	$stats{removed_prog_from_cache}++
239	} else {
240	$num_items++;
241	}
242	}
243	delete $data_cache->{prog}->{$key} if ($num_items == 0);
244	}
245
246	# age our title cache on startup
247	foreach my $key (keys %{($data_cache->{title})}) {
248	if ($data_cache->{title}->{$key}->{expires} < $script_start_time) {
249	delete $data_cache->{title}->{$key};
250	$stats{removed_title_from_cache}++
251	}
252	}
253	}
254	}
255
256	##############################################################################
257	# write out updated cache
258
259	sub write_cache
260	{
261	my $store;
262	$store->{data_cache} = $data_cache;
263	$store->{settings_override} = $settings_override;
264	$store->{failed_to_perform_lookups} = $stats{failed_to_perform_lookups};
265	Shepherd::Common::write_cache($opt->{cache_file}, $store);
266	}
267
268	##############################################################################
269
270	sub read_xmltv
271	{
272	my $filename = shift;
273	$d->{files} = 0 if (!defined $d->{files});
274
275	Shepherd::Common::log((sprintf " parsing: (%d) %s",$d->{files}+1,$filename));
276	$d->{data}->[$d->{files}] = XMLTV::parsefile($filename);
277
278	$d->{progcount}->[$d->{files}] = scalar(@{$d->{data}->[$d->{files}][3]});
279
280	$stats{programmes} += $d->{progcount}->[$d->{files}];
281	$d->{files}++;
282	}
283
284	##############################################################################
285
286	sub write_xmltv
287	{
288	Shepherd::Common::log("\nStage 5/5: writing output XMLTV into ".$opt->{output_file});
289
290	my %writer_args = ( encoding => 'ISO-8859-1' );
291	my $fh = new IO::File(">".$opt->{output_file}) \|\|
292	die "can't open $opt->{output_file} for writing: $!";
293	$writer_args{OUTPUT} = $fh;
294
295	my $writer = new XMLTV::Writer(%writer_args);
296	$writer->start( {
297	'source-info-url' => "http://thetvdb.com",
298	'source-info-name' => "$progname $version",
299	'generator-info-name' => "$progname $version"} );
300
301	# write channels out
302	for (my $i=0; $i < $d->{files}; $i++) {
303	my $chs = $d->{data}->[$i][2];
304	foreach my $ch (keys %$chs) {
305	if (!defined $d->{seen_channels}->{$ch}) {
306	$d->{seen_channels}->{$ch} = 1;
307	$writer->write_channel($d->{data}->[$i][2]->{$ch});
308	}
309	}
310	}
311
312	# write programmes out
313	for (my $i=0; $i < $d->{files}; $i++) {
314	for (my $j = 0; $j < $d->{progcount}->[$i]; $j++) {
315	my $prog = $d->{data}->[$i][3][$j];
316
317	my $title, my $subtitle;
318	$title = $prog->{title}->[0]->[0] if ((defined $prog->{title}) && (defined $prog->{title}->[0]) && (defined $prog->{title}->[0]->[0]));
319	my $progname = (defined $title ? $title : "");
320	my $added = "";
321	$title = lc($title) if ($title);
322	$subtitle = lc($prog->{'sub-title'}->[0]->[0]) if ((defined $prog->{'sub-title'}) && (defined $prog->{'sub-title'}->[0]) && (defined $prog->{'sub-title'}->[0]->[0]));
323
324	# If existing prog desc is short, see if we can add a better one
325	my $desc = (ref $prog->{desc} ? (ref $prog->{desc}->[0] ? $prog->{desc}->[0]->[0] : $prog->{desc}->[0]) : $prog->{desc});
326	if (!$desc or length ($desc) <= 80)
327	{
328	my $new_desc;
329	if ($title and $data_cache->{prog}->{$title})
330	{
331	if ($subtitle
332	and $data_cache->{prog}->{$title}->{$subtitle}
333	and $data_cache->{prog}->{$title}->{$subtitle}->{Overview}
334	and $data_cache->{prog}->{$title}->{$subtitle}->{Overview} ne "")
335	{
336	# Use 'Episode Info' as desc if there's currently no desc, or if
337	# the current desc is very short and Episode Info is longer.
338	if (!$desc or length($desc) < length($data_cache->{prog}->{$title}->{$subtitle}->{Overview}))
339	{
340	$new_desc = $data_cache->{prog}->{$title}->{$subtitle}->{Overview};
341	$added .= " replaced existing desc with Episode Info: '$new_desc'\n";
342	}
343	}
344	elsif ($data_cache->{prog}->{$title}->{SERIES}
345	and $data_cache->{prog}->{$title}->{SERIES}->{Overview}
346	and $data_cache->{prog}->{$title}->{SERIES}->{Overview} ne "")
347	{
348	# Use 'Series Info' as desc if there's nothing else
349	if ($desc) {
350	$new_desc = "$desc\n";
351	} else {
352	$new_desc = "";
353	}
354	$new_desc .= "Series Info: " . $data_cache->{prog}->{$title}->{SERIES}->{Overview};
355	$added .= " existing desc was blank or short, set to: '$new_desc'\n";
356	}
357	else
358	{
359	$added .= " existing desc blank or short but nothing to fill it with :(\n";
360	}
361
362	if ($new_desc)
363	{
364	$stats{inserted_new_title}++;
365	$prog->{desc}->[0]->[0] = $new_desc;
366	}
367	}
368	}
369
370
371	$desc = "";
372
373	# augment series data if we can
374	if ((defined $title) && (defined $data_cache->{prog}->{$title}) && (defined $data_cache->{prog}->{$title}->{SERIES})) {
375	$stats{augmented_prog_series_data}++;
376	my $series_data = $data_cache->{prog}->{$title}->{SERIES};
377
378	# description part
379	if ($opt->{long_info}) {
380	my $series_desc = "";
381	foreach my $field ("Status", "Overview", "FirstAired", "Network", "Genre", "Actors") {
382	$series_desc .= "\n ".$field.": ".$series_data->{$field} if ((defined $series_data->{$field}) && ($series_data->{$field} ne ""));
383	}
384	$desc .= "\nSeries Info:".$series_desc if ($series_desc ne "");
385	}
386
387	# Genre
388	if ((defined $series_data->{Genre}) && ($series_data->{Genre} ne "")) {
389	my @genres = split(/[\\|,]/, $series_data->{Genre});
390	foreach my $g (@genres) {
391	next unless ($g);
392	$g = Shepherd::Common::translate_category($g);
393	my $found_genre = 0;
394	foreach my $category (@{($prog->{category})}) {
395	$found_genre++ if (lc($g) eq lc($category->[0]));
396	}
397	if (!$found_genre) {
398	push(@{($prog->{category})},[$g]);
399	$added .= " added genre '$g'\n";
400	}
401	}
402	}
403
404	# Actors
405	# Not supplied by default because Series Actor info lists people
406	# who have EVER appeared in the show, which I think is
407	# misleading to present in MythTV's episode info.
408	if ($opt->{long_info} and $series_data->{Actors})
409	{
410	foreach my $cast (split(/[\\|,]/,$series_data->{Actors})) {
411	$cast =~ s/(^\s+\|\s+$)//g;
412	next if ($cast eq "");
413	my $found_cast = 0;
414	foreach my $a (@{($prog->{credits}->{actor})}) {
415	$found_cast++ if (lc($cast) eq lc($a));
416	}
417	if (!$found_cast) {
418	push(@{($prog->{credits}->{actor})},$cast);
419	Shepherd::Common::log(" added series actor '$cast'") if ($opt->{debug});
420	}
421	}
422	}
423	}
424
425
426	# augment episode data if we can
427	if ((defined $subtitle) && (defined $data_cache->{prog}->{$title}) && (defined $data_cache->{prog}->{$title}->{$subtitle})) {
428	$stats{augmented_prog_episode_data}++;
429	my $episode_data = $data_cache->{prog}->{$title}->{$subtitle};
430
431	# description part
432	if ($opt->{long_info}) {
433	my $episode_desc = "";
434	foreach my $field ("EpisodeNumber", "EpisodeName", "Overview", "ShowURL", "FirstAired", "GuestStars", "Director", "Writer", "DVD_discid", "DVD_season", "DVD_episodenumber", "DVD_chapter") {
435	$episode_desc .= "\n ".$field.": ".$episode_data->{$field} if ($episode_data->{$field});
436	}
437	$desc .= "\nEpisode Info:".$episode_desc if ($episode_desc ne "");
438	}
439
440	# ShowURL
441	if ((defined $episode_data->{ShowURL}) && ($episode_data->{ShowURL} ne "")) {
442	my $found_url = 0;
443	if (defined $prog->{url}) {
444	foreach my $url (@{($prog->{url})}) {
445	$found_url++ if (lc($url) eq lc($episode_data->{ShowURL}));
446	}
447	}
448	if (!$found_url) {
449	push (@{($prog->{url})},$episode_data->{ShowURL});
450	$added .= " added url '".$episode_data->{ShowURL}."'\n";
451	}
452	}
453
454	# GuestStars
455	#
456	# This is a little dodgy... it can include the full cast of the
457	# episode, even down to "Angry Nurse #3". Limited to 1 unless 'long-info'
458	if ((defined $episode_data->{GuestStars}) && ($episode_data->{GuestStars} ne "") &&
459	(defined $prog->{credits}) && (defined $prog->{credits}->{actor})) {
460	my $max_guests = ($opt->{long_info} ? 20 : 1);
461	foreach my $cast (split(/[\\|,]/,$episode_data->{GuestStars})) {
462	$cast =~ s/(^\s+\|\s+$)//g;
463	next if ($cast eq "");
464	my $found_cast = 0;
465	foreach my $a (@{($prog->{credits}->{actor})}) {
466	$found_cast++ if (lc($cast) eq lc($a));
467	}
468	if (!$found_cast) {
469	push(@{($prog->{credits}->{actor})},$cast);
470	$added .= " added guest star '$cast'\n";
471	$max_guests--;
472	last unless ($max_guests > 0);
473	}
474	}
475	}
476
477	# Director
478	if ((defined $episode_data->{Director}) && ($episode_data->{Director} ne "") &&
479	(defined $prog->{credits}) && (defined $prog->{credits}->{director})) {
480	foreach my $cast (split(/[\\|,]/,$episode_data->{Director})) {
481	$cast =~ s/(^\s+\|\s+$)//g;
482	next if ($cast eq "");
483	my $found_cast = 0;
484	foreach my $d (@{($prog->{credits}->{director})}) {
485	$found_cast++ if (lc($cast) eq lc($d));
486	}
487	if (!$found_cast) {
488	push(@{($prog->{credits}->{director})},$cast);
489	$added .= " added director '$cast'\n";
490	}
491	}
492	}
493
494	# Writer
495	if ((defined $episode_data->{Writer}) && ($episode_data->{Writer} ne "") &&
496	(defined $prog->{credits}) && (defined $prog->{credits}->{writer})) {
497	foreach my $cast (split(/[\\|,]/,$episode_data->{Writer})) {
498	$cast =~ s/(^\s+\|\s+$)//g;
499	next if ($cast eq "");
500	my $found_cast = 0;
501	foreach my $w (@{($prog->{credits}->{writer})}) {
502	$found_cast++ if (lc($cast) eq lc($w));
503	}
504	if (!$found_cast) {
505	push(@{($prog->{credits}->{writer})},$cast);
506	$added .= " added writer '$cast'\n";
507	}
508	}
509	}
510
511	# Episode Number
512	if ($episode_data->{EpisodeNumber})
513	{
514	my $xmltv_ns = ($episode_data->{SeasonNumber} ? ($episode_data->{SeasonNumber} - 1) : "") ." . ". ($episode_data->{EpisodeNumber} ? ($episode_data->{EpisodeNumber} - 1) : "") ." . 0";
515	$prog->{'episode-num'} = [ [ $xmltv_ns, 'xmltv_ns' ] ];
516	$added .= " added episode-num '$xmltv_ns'\n";
517	}
518	}
519
520	# should we add any text?
521	if ((!defined $opt->{dont_augment_desc}) && ($desc ne "")) {
522	$prog->{desc}->[0]->[0] = "" if (!defined $prog->{desc}->[0]->[0]);
523	$prog->{desc}->[0]->[0] .= "\n\n" if ($prog->{desc}->[0]->[0] ne "");
524	$prog->{desc}->[0]->[0] .= "TheTVDB.com augmented data:".$desc;
525	$added .= " augmented description\n";
526	}
527
528	if ($added ne '' and $opt->{debug})
529	{
530	chomp($added);
531	&Shepherd::Common::log(" - $progname\n$added")
532	}
533
534	Shepherd::Common::cleanup($prog);
535	$writer->write_programme($prog);
536	}
537	}
538
539	$writer->end();
540	}
541
542	##############################################################################
543	# get_url
544
545	sub get_url
546	{
547	my $url = shift;
548
549	my $data;
550	my $retries = $max_server_retries;
551	while (($data = &Shepherd::Common::get_url($url)) &&
552	($data =~ /This Account Has Exceeded Its PHP Quota/ \|\|
553	$data =~ /PHP Wrapper - 500 Server Error/) &&
554	$retries-- > 0) {
555	my $sleep = 93 + int(rand(120));
556	Shepherd::Common::log(" sleeping for $sleep seconds, This Account Has Exceeded Its PHP Quota...")
557	if ($data =~ /This Account Has Exceeded Its PHP Quota/);
558	Shepherd::Common::log(" sleeping for $sleep seconds, PHP Wrapper - 500 Server Error...")
559	if ($data =~ /PHP Wrapper - 500 Server Error/);
560	$stats{server_errors}++;
561	sleep($sleep);
562	$stats{slept_for} += $sleep;
563	}
564
565	return $data;
566	}
567
568	##############################################################################
569	# parser xml
570
571	sub parse
572	{
573	my $data = shift;
574
575	my $xml_tree;
576	eval { $xml_tree = $parser->parse($data); };
577	if (!$xml_tree) {
578	Shepherd::Common::log(" xml parse failed, attempting to fixup any missing ;");
579	if ($first_parse_error == 1) {
580	$first_parse_error = 0;
581	Shepherd::Common::log("$data");
582	}
583	# fix bad entities without ending ; by adding ; like "anc&#233, Trent"
584	$data =~ s/&(amp\|quot\|gt\|lt\|#\d+\|#x[0-9a-f]+)(?!;)/&$1;/gsi;
585
586	eval { $xml_tree = $parser->parse($data); };
587	if (!$xml_tree) {
588	Shepherd::Common::log(" xml parse of fixup failed, attempting to elimination any &");
589	# ignore bad entities without ending ; like "anc&#233, Trent"
590	$data =~ s/&/and/g;
591
592	eval { $xml_tree = $parser->parse($data); };
593	}
594	}
595
596	return $xml_tree;
597	}
598
599	##############################################################################
600	# process all xmltv files
601
602	sub perform_lookups
603	{
604	$d->{series_lookup_requests} = 0;
605	$d->{episode_lookup_requests} = 0;
606
607	Shepherd::Common::log("\nStage 2/5: processing ".$stats{programmes}." programmes ...");
608	my $prog_count = 0;
609	my $last_updated;
610	for (my $i=0; $i < $d->{files}; $i++) {
611	for (my $j = 0; $j < $d->{progcount}->[$i]; $j++) {
612	$prog_count++;
613	if ((!$last_updated) \|\| ((time - $last_updated) > 20) \|\| ($prog_count == $stats{programmes})) {
614	$last_updated = time;
615	my $percent_complete = (($prog_count+1) / ($stats{programmes}+1));
616	my $estimaged_time = ((time - $script_start_time) / $percent_complete);
617	Shepherd::Common::log((sprintf " .. at programme %d of %d (%0.1f%%) [%s elapsed] ..",
618	$prog_count, $stats{programmes}, ($percent_complete * 100),
619	Shepherd::Common::pretty_duration((time - $script_start_time))));
620	}
621
622	my $prog = $d->{data}->[$i][3][$j];
623	my $title, my $subtitle;
624	$title = $prog->{title}->[0]->[0] if ((defined $prog->{title}) && (defined $prog->{title}->[0]) && (defined $prog->{title}->[0]->[0]));
625	$subtitle = $prog->{'sub-title'}->[0]->[0] if ((defined $prog->{'sub-title'}) && (defined $prog->{'sub-title'}->[0]) && (defined $prog->{'sub-title'}->[0]->[0]));
626
627	if ((defined $title) && (include_prog($prog))) {
628	my $lc_title = lc($title);
629	&lookup_title_web($lc_title) if (!defined $data_cache->{prog}->{$lc_title});
630
631	# proceed to episode lookup if we have something in our title cache
632	if ((defined $data_cache->{title}->{$lc_title}) && ($data_cache->{title}->{$lc_title}->{notfound} == 0)) {
633	$subtitle = "" if (!defined $subtitle);
634	&lookup_episode($lc_title, $subtitle);
635	}
636	}
637	}
638	}
639
640	Shepherd::Common::log("\nStage 3/5: processing ".$d->{series_lookup_requests}." series detail lookup requests ...");
641	&lookup_series_updates if ($d->{series_lookup_requests} > 0);
642
643	Shepherd::Common::log("\nStage 4/5: processing ".$d->{episode_lookup_requests}." episode detail lookup requests ...");
644	&lookup_episode_updates if ($d->{episode_lookup_requests} > 0);
645	}
646
647	##############################################################################
648	# helper routine: returns 1 if we should look up programme, 0 if not
649
650	sub include_prog
651	{
652	my $prog = shift;
653
654	if ((!defined $prog->{title}) \|\| (!defined $prog->{title}->[0]) \|\| (!defined $prog->{title}->[0]->[0])) {
655	$stats{skipped_due_to_title}++;
656	next;
657	}
658	my $title = $prog->{title}->[0]->[0];
659
660	# skip station close
661	if (($title =~ /^close$/i) \|\| ($title =~ /^station close$/i)) {
662	$stats{skipped_due_to_title}++;
663	return 0;
664	}
665
666	# skip categories
667	if (defined $prog->{category}) {
668	foreach my $prog_category (@{($prog->{category})}) {
669	foreach my $prog_cat2 (@$prog_category) {
670	foreach my $skip_category (split(/,/,$opt->{skip_categories})) {
671	if (lc($prog_cat2) eq lc($skip_category)) {
672	$stats{skipped_due_to_category}++;
673	return 0;
674	}
675	}
676	}
677	}
678	}
679
680	# only lookup if min_duration < prog_duration > min_duration
681	my $t1 = Shepherd::Common::parse_xmltv_date($prog->{start});
682	my $t2 = Shepherd::Common::parse_xmltv_date($prog->{stop});
683	if ((!$t1) \|\| (!$t2)) {
684	$stats{excluded_couldnt_parse_time}++;
685	return 0;
686	}
687
688	# ensure prog is within duration limits
689	my $prog_duration = (($t2 - $t1) / 60);
690	if (($prog_duration < $opt->{min_duration}) \|\| ($prog_duration > $opt->{max_duration})) {
691	$stats{excluded_prog_duration}++;
692	return 0;
693	}
694
695	return 1;
696	}
697
698	##############################################################################
699	# fill in $d->{tvdb}->{mirrors} with a list of tvdb mirror sites
700
701	sub lookup_mirrors
702	{
703	my $data = &get_url($mirrorlist_url);
704	die "could not gather list of mirrors" if (!$data);
705	my $xml_tree = &parse($data);
706	die "could not parse list of mirrors" if (!$xml_tree);
707
708	$d->{tvdb}->{num_mirrors} = $xml_tree->getElementsByTagName("Item")->getLength;
709	for (my $i = 0; $i < $d->{tvdb}->{num_mirrors}; $i++) {
710	push(@{($d->{tvdb}->{mirrors})},
711	$xml_tree->getElementsByTagName("Item")->item($i)->getElementsByTagName("interface")->item(0)->getFirstChild->getNodeValue);
712	}
713	$xml_tree->dispose;
714
715	die "no mirrors found" if ($d->{tvdb}->{num_mirrors} == 0);
716
717	$d->{tvdb}->{mirror} = $d->{tvdb}->{mirrors}[int(rand($d->{tvdb}->{num_mirrors}))];
718	Shepherd::Common::log(" chose mirror ".$d->{tvdb}->{mirror}." for data");
719
720	}
721
722	##############################################################################
723	# find a 'seriesid' associated with this title
724	# (actually just populates $data_cache->{title}->{$lc_title})
725
726	sub lookup_title_web
727	{
728	my ($lc_title) = @_;
729
730	my $letter = substr($lc_title,0,1);
731	$letter = "OTHER" if ($letter !~ /[a-z]/);
732
733	# only lookup if we have passed our caching threshold for this letter...
734	if (!defined $data_cache->{title}->{"WEB_LOOKUP_".$letter}) {
735	Shepherd::Common::log(" fetching series tables for '".$letter."' ...");
736
737	my $url = "http://thetvdb.com/?tab=listseries&letter=".$letter;
738	my $data = &get_url($url);
739	if (!$data) {
740	Shepherd::Common::log(" ".$url." didn't return any valid data! skipping for 7 days...");
741
742	# try again in 7 days
743	$data_cache->{title}->{"WEB_LOOKUP_".$letter}->{expires} = $script_start_time + (7 * 86400);
744	$stats{failed_title_web_fetch}++;
745	die "failed title web fetch" if $stats{failed_title_web_fetch} >= $max_failures;
746	return;
747	}
748
749	my $expires_in = $script_start_time + ($opt->{cache_title_for} * 86400);
750	$data_cache->{title}->{"WEB_LOOKUP_".$letter}->{expires} = $expires_in;
751
752	$data = Encode::decode_utf8($data);
753	my $tree = HTML::TreeBuilder->new_from_content($data);
754	my $tree_table = $tree->look_down('_tag' => 'table', 'id' => 'listtable');
755
756	foreach my $tree_tr ($tree_table->look_down('_tag' => 'tr')) {
757	my @tree_td = $tree_tr->look_down('_tag' => 'td');
758
759	if (((scalar @tree_td) == 3) && ($tree_td[2]->as_text() =~ /^(\d+)$/)) {
760	my $series_name = lc($tree_td[0]->as_text());
761	$data_cache->{title}->{$series_name}->{SeriesName} = $tree_td[0]->as_text();
762	$data_cache->{title}->{$series_name}->{SeriesID} = $tree_td[2]->as_text();
763	$data_cache->{title}->{$series_name}->{expires} = $expires_in;
764	$data_cache->{title}->{$series_name}->{notfound} = 0;
765	$stats{inserted_title_into_cache}++;
766	}
767	}
768
769	&write_cache;
770	$tree->delete;
771	}
772	}
773
774
775	##############################################################################
776	# find an 'episode' associated with a series
777
778	sub lookup_episode
779	{
780	my ($lc_title, $subtitle) = @_;
781	my $seriesid = $data_cache->{title}->{$lc_title}->{SeriesID};
782	my $lc_subtitle = lc($subtitle);
783	return if (!defined $seriesid);
784
785	$stats{used_title_cache_item}++;
786
787	my $url = "/GetEpisodes.php?seriesid=".$seriesid."&IncludeSeriesInfo=1";
788	if ($lc_subtitle eq "") {
789	$url .= "&season=1&episode=1";
790	$lc_subtitle = "NONE";
791	} else {
792	$url .= "&episodename=".Shepherd::Common::urlify($lc_subtitle);
793	}
794
795	# return without doing anything if the entry already exists in the cache
796	if (defined $data_cache->{prog}->{$lc_title}->{$lc_subtitle}) {
797	$stats{used_prog_cache_item}++;
798	goto CHECK_EPISODE if ($lc_subtitle ne "NONE");
799	return;
800	}
801
802	Shepherd::Common::log(" fetching series '".$data_cache->{title}->{$lc_title}->{SeriesName}."' ".($lc_subtitle ne "NONE" ? "(episode '$subtitle')" : ""). " ...");
803
804	&lookup_mirrors if (!defined $d->{tvdb}->{mirror});
805	my $data = &get_url($d->{tvdb}->{mirror}.$url);
806	if (!$data) {
807	Shepherd::Common::log(" series lookup of ".$d->{tvdb}->{mirror}.$url." failed");
808	$stats{prog_lookup_failed}++;
809	die "prog lookup failed" if $stats{prog_lookup_failed} >= $max_failures;
810	return;
811	}
812	my $xml_tree = &parse($data);
813	if (!$xml_tree) {
814	Shepherd::Common::log(" series parse of ".$d->{tvdb}->{mirror}.$url." failed");
815	$stats{prog_lookup_failed}++;
816	die "prog lookup failed" if $stats{prog_lookup_failed} >= $max_failures;
817	return;
818	}
819
820	my $num_items = $xml_tree->getElementsByTagName("Item")->getLength;
821
822	if ($num_items == 1) {
823	$stats{negatively_cached_prog}++;
824
825	$data_cache->{prog}->{$lc_title}->{$lc_subtitle}->{notfound} = 1;
826	$data_cache->{prog}->{$lc_title}->{$lc_subtitle}->{expires} = $script_start_time + (($opt->{cache_details_for} / 2) * 86400);
827
828	# on an episode-lookup failure, thetvdb.com doesn't return any SeriesData,
829	# so schedule that for a bulk get.
830	if ((!defined $data_cache->{prog}->{$lc_title}->{SERIES}) &&
831	(!defined $d->{series_lookup}->{$lc_title})) {
832	$d->{series_lookup}->{$lc_title} = $seriesid;
833	$d->{series_lookup_requests}++;
834	}
835	return;
836	}
837
838	my $item = $xml_tree->getElementsByTagName("Item")->item(1);
839
840	# remove existing entries
841	delete $data_cache->{prog}->{$lc_title}->{SERIES} if (defined $data_cache->{prog}->{$lc_title}->{SERIES});
842	delete $data_cache->{prog}->{$lc_title}->{$lc_subtitle} if (defined $data_cache->{prog}->{$lc_title}->{$lc_subtitle});
843
844	# set expiry on these
845	$data_cache->{prog}->{$lc_title}->{SERIES}->{expires} = $script_start_time + ($opt->{cache_details_for} * 86400);
846	$data_cache->{prog}->{$lc_title}->{$lc_subtitle}->{expires} = $script_start_time + ($opt->{cache_details_for} * 86400);
847
848	foreach my $field ("SeriesData-Status", "SeriesData-FirstAired", "SeriesData-Network", "SeriesData-Genre", "SeriesData-Actors", "SeriesData-Overview", "id", "SeasonNumber", "EpisodeNumber", "EpisodeName") {
849	my $fieldtag = $item->getElementsByTagName($field)->item(0)->getFirstChild;
850	if (defined $fieldtag) {
851	if ($field =~ /^SeriesData-(.*)$/) {
852	$data_cache->{prog}->{$lc_title}->{SERIES}->{$1} = $fieldtag->getNodeValue;
853	} else {
854	$data_cache->{prog}->{$lc_title}->{$lc_subtitle}->{$field} = $fieldtag->getNodeValue;
855	}
856	}
857	}
858	$xml_tree->dispose;
859	$stats{inserted_prog_into_cache}++;
860	&write_cache if (($stats{inserted_prog_into_cache} % 15) == 0);
861
862	CHECK_EPISODE:
863	if ((defined $data_cache->{prog}->{$lc_title}->{$lc_subtitle}->{id}) &&
864	(!defined $data_cache->{prog}->{$lc_title}->{$lc_subtitle}->{have_episode_details})) {
865	my $ep;
866	$ep->{id} = $data_cache->{prog}->{$lc_title}->{$lc_subtitle}->{id};
867	$ep->{title} = $lc_title;
868	$ep->{subtitle} = $lc_subtitle;
869
870	push(@{($d->{episode_lookup})}, $ep);
871	$d->{episode_lookup_requests}++;
872	}
873	}
874
875	##############################################################################
876	# used to lookup SeriesData for known (good) seriesid's
877
878	sub lookup_series_updates
879	{
880	my @id_list = values %{($d->{series_lookup})};
881
882	while ((scalar @id_list) > 0) {
883	Shepherd::Common::log(" ".(scalar @id_list)." remaining...");
884
885	&lookup_mirrors if (!defined $d->{tvdb}->{mirror});
886	my $url = $d->{tvdb}->{mirror}."/SeriesUpdates.php?lasttime=0&idlist=";
887
888	# grab up to 20 at a time
889	foreach my $count (1..20) {
890	my $id = pop(@id_list);
891
892	if (defined $id) {
893	$url .= ',' if ($count > 1);
894	$url .= $id;
895	}
896	}
897
898	my $data = &get_url($url);
899	if (!$data) {
900	Shepherd::Common::log(" series detail lookup request of ".$url." failed");
901	$stats{series_update_lookup_failed}++;
902	die "series update lookup failed" if $stats{series_update_lookup_failed} >= $max_failures;
903	next;
904	}
905	my $xml_tree = &parse($data);
906	if (!$xml_tree) {
907	Shepherd::Common::log(" series detail lookup parse of ".$url." failed");
908	$stats{series_update_lookup_failed}++;
909	die "series update lookup failed" if $stats{series_update_lookup_failed} >= $max_failures;
910	next;
911	}
912
913	my $num_items = $xml_tree->getElementsByTagName("Item")->getLength;
914
915	for (my $i=1; $i < $num_items; $i++) {
916	my $item = $xml_tree->getElementsByTagName("Item")->item($i);
917	my $namefield = $item->getElementsByTagName("SeriesName")->item(0)->getFirstChild;
918
919	if (defined $namefield) {
920	my $name = lc($namefield->getNodeValue);
921
922	foreach my $field ("Status", "FirstAired", "Network", "Genre", "Actors", "Overview") {
923	my $fieldtag = $item->getElementsByTagName($field)->item(0)->getFirstChild;
924	if (defined $fieldtag) {
925	$data_cache->{prog}->{$name}->{SERIES}->{$field} = $fieldtag->getNodeValue;
926	}
927	}
928
929	$data_cache->{prog}->{$name}->{SERIES}->{expires} = $script_start_time + ($opt->{cache_details_for} * 86400);
930	$stats{inserted_prog_into_cache}++;
931	&write_cache if (($stats{inserted_prog_into_cache} % 15) == 0);
932	}
933	}
934
935	$xml_tree->dispose;
936	}
937
938	&write_cache;
939	}
940
941	##############################################################################
942	# used to lookup episode details
943
944	sub lookup_episode_updates
945	{
946	my $episodes;
947
948	while ((scalar @{($d->{episode_lookup})}) > 0) {
949	Shepherd::Common::log(" ".(scalar @{($d->{episode_lookup})})." remaining...");
950
951	&lookup_mirrors if (!defined $d->{tvdb}->{mirror});
952	my $url = $d->{tvdb}->{mirror}."/EpisodeUpdates.php?lasttime=0&idlist=";
953
954	# grab up to 20 at a time
955	foreach my $count (1..20) {
956	my $ep = pop(@{($d->{episode_lookup})});
957
958	if (defined $ep) {
959	my $id = $ep->{id};
960	$episodes->{$id}->{title} = $ep->{title};
961	$episodes->{$id}->{subtitle} = $ep->{subtitle};
962
963	$url .= ',' if ($count > 1);
964	$url .= $id;
965	}
966	}
967
968	my $data = &get_url($url);
969	if (!$data) {
970	Shepherd::Common::log(" episode detail lookup request of ".$url." failed");
971	$stats{episode_update_lookup_failed}++;
972	die "episode update lookup failed" if $stats{episode_update_lookup_failed} >= $max_failures;
973	next;
974	}
975	my $xml_tree = &parse($data);
976	if (!$xml_tree) {
977	Shepherd::Common::log(" episode detail lookup parse of ".$url." failed");
978	$stats{episode_update_lookup_failed}++;
979	die "episode update lookup failed" if $stats{episode_update_lookup_failed} >= $max_failures;
980	next;
981	}
982
983	my $num_items = $xml_tree->getElementsByTagName("Item")->getLength;
984
985	for (my $i=1; $i < $num_items; $i++) {
986	my $item = $xml_tree->getElementsByTagName("Item")->item($i);
987	my $id_field = $item->getElementsByTagName("id")->item(0);
988
989	if ((defined $id_field) && (defined $id_field->getFirstChild)) {
990	my $id = $id_field->getFirstChild->getNodeValue;
991	my $title = $episodes->{$id}->{title};
992	my $subtitle = $episodes->{$id}->{subtitle};
993
994	foreach my $field ("id", "EpisodeNumber", "EpisodeName", "FirstAired", "GuestStars", "Director", "Writer", "Overview", "ShowURL", "DVD_discid", "DVD_season", "DVD_episodenumber", "DVD_chapter", "IncorrectID") {
995	# print "looking for $field in item $i $url\n";
996	my $fieldtag = $item->getElementsByTagName($field)->item(0);
997	if ((defined $fieldtag) && (defined $fieldtag->getFirstChild)) {
998	$data_cache->{prog}->{$title}->{$subtitle}->{$field} = $fieldtag->getFirstChild->getNodeValue;
999	}
1000	}
1001	$data_cache->{prog}->{$title}->{$subtitle}->{have_episode_details} = 1;
1002	$data_cache->{prog}->{$title}->{$subtitle}->{expires} = $script_start_time + (($opt->{cache_details_for} / 2) * 86400);
1003
1004	$stats{inserted_episode_into_cache}++;
1005	&write_cache if (($stats{inserted_episode_into_cache} % 15) == 0);
1006	}
1007	}
1008
1009	$xml_tree->dispose;
1010	}
1011
1012	&write_cache;
1013	}
1014
1015	##############################################################################

Note: See TracBrowser for help on using the browser.

Context Navigation

root/trunk/postprocessors/tvdb_augment_data

Download in other formats: