Context Navigation

yahoo7widget @ 73

Revision 73, 16.6 kB (checked in by lincoln, 7 years ago)

variety of bug fixes and enhancements

(1) yahoo7widget:

fill in extra fields correctly (cast/credits/categories/date/ratings etc)
use the same categories translation table as what Max has in Rex
don't output times with any timezone

(2) reconciler_mk2:

fix some XMLTV parsing so as to output some of the more obscure fields correctly
DateTime::Format::Strptime was actually parsing timezones when we weren't asking it to, skewing all dates. roll our own parse_xmltv_date to get around such bogosity

Property svn:executable set to *

Rev	Line
[2]	1	#!/usr/bin/perl -w
	2
	3	# yahoo7_widget au_tv guide grabber - runs from "Shepherd" master grabber
	4	# * written by ltd
	5	# * uses yahoo7 widget for ABC/7/9/10/SBS (all they have)
	6	# * when used in conjunction with Shepherd, shepherd can collect other channels
	7	# using other grabbers
	8	# * this does NOT use any config file - all settings are passed in from shepherd
	9
	10	# changelog:
	11	# 1.50 22sep06 added support for "shepherd" master grabber script
	12	# 1.51 02oct06 --ready option
	13	# 1.52 03oct06 split out yahoo7 grabber into its own grabber
[72]	14	# 1.54 16oct06 put date/cast/credits/year into correct xmltv fields
[2]	15
	16	use strict;
	17
	18	my $progname = "yahoo7widget";
[72]	19	my $version = "1.54_16oct06";
[2]	20
	21	use LWP::UserAgent;
	22	use Time::HiRes qw(gettimeofday tv_interval);
	23	use XMLTV;
	24	use XML::DOM;
	25	use XML::DOM::NodeList;
	26	use POSIX qw(strftime mktime);
	27	use Getopt::Long;
	28	use Data::Dumper;
	29	use Cwd;
	30
	31	#
	32	# some initial cruft
	33	#
	34
	35	my $script_start_time = [gettimeofday];
	36	my %stats;
	37	my $channels;
	38	my $tv_guide;
	39
	40	# lets make sure we look exactly like the yahoo widget engine...
	41	my $ua;
	42	BEGIN {
	43	$ua = LWP::UserAgent->new(
	44	'timeout' => 30,
	45	'keep_alive' => 1,
	46	'agent' => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-us)'
	47	);
	48	$ua->env_proxy;
	49	# $ua->cookie_jar({});
	50	$\| = 1;
	51	}
	52
	53	#
	54	# parse command line
	55	#
	56
	57	my $opt_days = 7; # default
	58	my $opt_offset = 0; # default
	59	my $opt_timezone = "1000"; # default
	60	my $opt_outputfile = cwd() . "/yahoo7widget.xmltv"; # default
	61	my $opt_configfile = cwd() . "/yahoo7widget.conf"; # ignored
	62	my $opt_channels_file= "";
	63	my $opt_fast = 0;
	64	my $opt_warper = 0;
	65	my $opt_obfuscate = 0;
	66	my $opt_help = 0;
	67	my $opt_version = 0;
	68	my $opt_desc = 0;
	69	my $opt_dont_retry = 0;
	70	my $debug = 1;
	71	my $lang = "en";
	72	my $region = 94;
	73	my $time_offset = 0;
[28]	74	my $opt_raw;
[2]	75
	76	GetOptions(
	77	'region=i' => \$region,
	78	'days=i' => \$opt_days,
	79	'offset=i' => \$opt_offset,
	80	'timezone=s' => \$opt_timezone,
	81	'channels_file=s' => \$opt_channels_file,
	82	'output=s' => \$opt_outputfile,
	83	'config-file=s' => \$opt_configfile,
	84	'fast' => \$opt_fast,
	85	'debug+' => \$debug,
	86	'warper' => \$opt_warper,
	87	'lang=s' => \$lang,
	88	'obfuscate' => \$opt_obfuscate,
	89	'no-retry' => \$opt_dont_retry,
	90	'help' => \$opt_help,
	91	'verbose' => \$opt_help,
	92	'version' => \$opt_version,
	93	'ready' => \$opt_version,
	94	'desc' => \$opt_desc,
[28]	95	'raw=s' => \$opt_raw,
[2]	96	'v' => \$opt_help);
	97
	98	&help if ($opt_help);
	99
	100	if ($opt_version \|\| $opt_desc) {
	101	printf "%s %s\n",$progname,$version;
	102	printf "%s is a details-aware grabber that collects very high quality data (full title/subtitle/description/genre and year/cast/credits data) using the Yahoo7 widget for ABC/7/9/10/SBS only.",$progname if $opt_desc;
	103	exit(0);
	104	}
	105
	106	die "no channel file specified, see --help for instructions\n", if ($opt_channels_file eq "");
	107
	108	#
	109	# go go go!
	110	#
	111
	112	# normalize starttime to an hour..
	113	my $starttime = time;
	114	my ($sec,$min,@rest) = localtime($starttime);
	115	$starttime -= ((60 * $min) + $sec);
	116	my $endtime = $starttime + ($opt_days * 86400);
	117	$starttime += (86400 * $opt_offset);
	118
	119	&log(sprintf "going to grab %d days%s of data into %s (%s%s) timezone %s region %s",
	120	$opt_days,
	121	($opt_offset ? " (skipping first %d days)" : ""),
	122	$opt_outputfile,
	123	($opt_fast ? "with haste" : "slowly"),
	124	($opt_warper ? ", anonymously" : ""),
	125	$opt_timezone, $region);
	126
	127	$time_offset = 3600((($opt_timezone / 100)-10) + (0.1 (($opt_timezone % 100) / 60))) if ($opt_timezone ne "1000");
	128
	129	if (-r $opt_channels_file) {
	130	local (@ARGV, $/) = ($opt_channels_file);
	131	no warnings 'all'; eval <>; die "$@" if $@;
	132	} else {
	133	die "WARNING: channels file $opt_channels_file could not be read\n";
	134	}
	135
[28]	136	unlink $opt_raw if ($opt_raw);
	137
[2]	138	for (my $currtime = $starttime; $currtime < $endtime; $currtime += 86400) {
	139	&parse_xml_data( get_url(
	140	(sprintf "http://au.tv.yahoo.com/widget.html?rg=%d&st=%d&et=%d", $region, $currtime,($currtime+86400)),
[4]	141	(sprintf "yahoo7widget detailed data: day %d of %d",((($currtime-$starttime)/86400)+1),(($endtime-$starttime)/86400))));
[2]	142	}
	143
	144	&write_data;
	145	&print_stats;
	146	exit(0);
	147
	148	######################################################################################################
	149	# help
	150
	151	sub help
	152	{
	153	print<<EOF
	154	$progname $version
	155
	156	options are as follows:
	157	--help show these help options
	158	--days=N fetch 'n' days of data (default: $opt_days)
	159	--output=file send xml output to file (default: "$opt_outputfile")
	160	--config-file=file (ignored - historically used by grabbers not not this one)
	161	--fast don't run slow - get data as quick as you can - not recommended
	162	--debug increase debug level
	163	--warper fetch data using WebWarper web anonymizer service
	164	--obfuscate pretend to be a proxy servicing multiple clients
	165	--no-retry if webserver is rejecting our request, don't retry (default: do retry)
	166	--lang=[s] set language of xmltv output data (default $lang)
	167
	168	--region=N set region for where to collect data from (default: $region)
	169	--channels_file=file where to get channel data from (MANDATORY)
	170	--timezone=HHMM timezone for channel data (default: $opt_timezone)
	171	EOF
	172	;
	173
	174	exit(0);
	175	}
	176
	177	######################################################################################################
	178	# transcode ywe-octet-stream back into text
	179
	180	sub transform_output
	181	{
	182	my ($datasize, $data) = @_;
	183
	184	my @xform_map = (
	185	0x39, 0x9E, 0x05, 0x72, 0x6C, 0x06, 0x38, 0x15, 0x42, 0x1E, 0xB9, 0xFD, 0x4D, 0x08, 0x0C, 0x2E,
	186	0x57, 0xC7, 0x62, 0x6E, 0xC5, 0x3A, 0x3C, 0xA4, 0x1D, 0xC6, 0x3D, 0x18, 0x2D, 0x1B, 0x83, 0x20,
	187	0x78, 0xFC, 0xA5, 0xDE, 0x28, 0xE8, 0x3E, 0x9B, 0x7C, 0x22, 0x1C, 0x89, 0xFF, 0x52, 0x54, 0x43,
	188	0x51, 0x7F, 0x71, 0x40, 0x7A, 0xCF, 0x65, 0xE4, 0x36, 0xEB, 0xC9, 0x1F, 0x80, 0x9A, 0x31, 0x4A,
	189	0x45, 0xD4, 0x2B, 0x02, 0x4C, 0xF4, 0x53, 0xBD, 0xA8, 0xF9, 0x50, 0x61, 0x8A, 0xD5, 0xBF, 0x81,
	190	0xC0, 0xDB, 0xFE, 0xF7, 0xBA, 0xEC, 0xFA, 0x73, 0xA9, 0x8F, 0xB1, 0x70, 0x33, 0xCE, 0x60, 0xAC,
	191	0xB2, 0x58, 0x26, 0x85, 0x6B, 0x7D, 0x93, 0x03, 0x64, 0x47, 0x04, 0x88, 0x01, 0xA6, 0x3B, 0x90,
	192	0x98, 0xF5, 0x97, 0x3F, 0xF6, 0xD3, 0x94, 0xB7, 0x29, 0x07, 0x96, 0x6F, 0x14, 0x35, 0x8D, 0x2A,
	193	0x16, 0x17, 0x8B, 0xD1, 0x48, 0xD6, 0xF1, 0xE2, 0x79, 0x2C, 0x41, 0x5B, 0xBC, 0xB5, 0x68, 0xDC,
	194	0x49, 0xD2, 0x6A, 0xCC, 0x25, 0xB4, 0xAA, 0x63, 0x9C, 0x56, 0x4B, 0xB8, 0x87, 0x5E, 0x86, 0x09,
	195	0xC4, 0x95, 0xB6, 0x12, 0xF8, 0x84, 0x4E, 0x21, 0x32, 0xCA, 0x66, 0xC3, 0xBB, 0x27, 0xEE, 0xE0,
	196	0x1A, 0xD8, 0x6D, 0x4F, 0xAF, 0x82, 0xEF, 0xCD, 0x5F, 0x8C, 0x67, 0xA2, 0xCB, 0xED, 0xAB, 0xB0,
	197	0xA7, 0x92, 0x75, 0x5A, 0xF2, 0x0A, 0x0E, 0xE6, 0x7E, 0xC8, 0xE9, 0x19, 0x24, 0x37, 0x11, 0xA0,
	198	0xE3, 0xDD, 0xD7, 0x23, 0x9F, 0x00, 0xA1, 0xC1, 0x74, 0xF0, 0x99, 0x77, 0xAE, 0x91, 0x7B, 0xFB,
	199	0xD9, 0xDA, 0xC2, 0x44, 0x0D, 0x76, 0x10, 0x9D, 0xEA, 0xE7, 0xE5, 0x59, 0xF3, 0xD0, 0x5D, 0x2F,
	200	0x69, 0xAD, 0x34, 0x0F, 0x5C, 0x8E, 0xBE, 0x13, 0x30, 0x55, 0xE1, 0xDF, 0x0B, 0xB3, 0x46, 0xA3);
	201	my ($xlate_pos1, $xlate_pos2, $xlate_pos3, $xlate_pos4) = (0,0,0,0);
	202	my $outputdata;
	203
	204	if (($datasize < 1) \|\| (ord(substr($data,0,1)) != 1)) {
	205	# not valid
	206	return(undef);
	207	}
	208
	209	for (my $pos = 1; $pos < $datasize; $pos++) {
	210	$xlate_pos1 = ($xlate_pos1 + 1) % 256;
	211	$xlate_pos3 = $xform_map[$xlate_pos1];
	212	$xlate_pos4 = ($xlate_pos2 + $xlate_pos3) % 256;
	213	$xlate_pos2 = $xform_map[$xlate_pos4];
	214	$xform_map[$xlate_pos1] = $xlate_pos2;
	215	$xlate_pos2 += $xlate_pos3;
	216	$xform_map[$xlate_pos4] = $xlate_pos3;
	217	$xlate_pos2 = $xlate_pos2 % 256;
	218	$xlate_pos3 = $xform_map[$xlate_pos2];
	219	$xlate_pos2 = $xlate_pos4;
	220	$outputdata .= chr((((ord(substr($data,$pos,1))) % 256) ^ ($xlate_pos3 % 256)) % 256);
	221	}
[28]	222
	223	if ($opt_raw) {
	224	if (open(F,">>$opt_raw")) {
	225	print F $outputdata;
	226	close F;
	227	}
	228	}
	229
[2]	230	return($outputdata);
	231	}
	232
	233	######################################################################################################
	234	# logic to fetch a page via http
	235	# retries up to 3 times to get a page with 5 second pauses inbetween
	236
	237	sub get_url
	238	{
	239	my ($url,$status,$dontretry) = @_;
	240	my $response;
	241	my $attempts = 0;
	242	my ($raw, $page, $base);
	243
	244	$url =~ s#^http://#http://webwarper.net/ww/# if $opt_warper;
	245	my $request = HTTP::Request->new(GET => $url);
	246	$request->header('Accept-Encoding' => 'gzip');
	247
	248	if ($opt_obfuscate) {
	249	my $randomaddr = sprintf "203.%d.%d.%d",rand(255),rand(255),(rand(254)+1);
	250	$request->header('Via' => '1.0 proxy:81 (Squid/2.3.STABLE3)');
	251	$request->header('X-Forwarded-For' => $randomaddr);
	252	}
[4]	253	&log(sprintf "fetching %s%s: %s",$status,($opt_obfuscate ? "[obfuscate]" : ""),$url);
[2]	254	for (1..3) {
	255	$response = $ua->request($request);
	256	last if ($response->is_success \|\| $dontretry);
	257
	258	$stats{http_failed_requests}++;
	259	$stats{slept_for} += 10;
	260	$attempts++;
	261	sleep 10;
	262	}
	263	if (!($response->is_success)) {
	264	if ($dontretry == 0) {
	265	&log("aborting after $attempts attempts to fetch url $url") if $debug;
	266	printf STDERR "ERROR: could not open url %s in %d attempts\n",$url,$attempts;
	267	}
	268	return undef;
	269	}
	270
	271	$stats{bytes_fetched} += do {use bytes; length($response->content)};
	272	$stats{http_successful_requests}++;
	273
	274	if (!$opt_fast) {
	275	my $sleeptimer = int(rand(5)) + 1; # sleep anywhere from 1 to 5 seconds
	276	$stats{slept_for} += $sleeptimer;
	277	sleep $sleeptimer;
	278	}
	279
	280	if ($response->header('Content-Encoding') &&
	281	$response->header('Content-Encoding') eq 'gzip') {
	282	$stats{compressed_pages} += do {use bytes; length($response->content)};
	283	$response->content(Compress::Zlib::memGunzip($response->content));
	284	}
	285
	286	if ($response->header('Content-type') eq 'xapplication/ywe-octet-stream') {
	287	$stats{transformed_pages}++;
	288	$base = &transform_output(length($response->content), $response->content);
	289	} else {
	290	$base = $response->content;
	291	}
	292	return $base;
	293	}
	294
	295	######################################################################################################
	296
	297	sub log
	298	{
	299	my ($entry) = @_;
[4]	300	printf STDERR "%s [%d] %s\n",$progname,time,$entry;
[2]	301	}
	302
	303	######################################################################################################
	304
	305	sub print_stats
	306	{
[4]	307	printf STDERR "%s v%s [%d] completed in %0.2f seconds",$progname,$version,time,tv_interval($script_start_time);
[2]	308	foreach my $key (sort keys %stats) {
	309	printf STDERR ", %d %s",$stats{$key},$key;
	310	}
	311	printf STDERR "\n";
	312	}
	313
	314	######################################################################################################
	315	# given yahoo7 xml data, parse it into 'shows' ..
	316	# parse it into $tv_guide->{$channel}->{data}->{$event_id}-> structures..
	317
	318	sub parse_xml_data
	319	{
	320	my $data = shift;
	321	my $parser = new XML::DOM::Parser;
	322	my $tree = $parser->parse($data);
	323	my $tree_channels = $tree->getElementsByTagName("venue");
	324	for (my $i = 0; $i < $tree_channels->getLength; $i++) {
	325	my $channel = $tree_channels->item($i)->getAttributeNode("co_short")->getValue;
	326
	327	# for this channel get every programme ('event')
	328	my $events = $tree_channels->item($i)->getElementsByTagName("event");
	329	for (my $j = 0; $j < $events->getLength; $j++) {
	330	my $event = $events->item($j);
	331	my $event_id = $event->getElementsByTagName("event_id")->item(0)->getFirstChild->getNodeValue;
	332
	333	# mandatory fields
	334	my $event_start = $event->getElementsByTagName("event_date")->item(0)->getFirstChild->getNodeValue;
	335	my $event_end = $event->getElementsByTagName("end_date")->item(0)->getFirstChild->getNodeValue;
[73]	336	$event_id .= $event_start . $event_end; # event_id actually isn't unique - so make it so
[2]	337
[73]	338	$stats{programmes}++;
	339	$stats{duplicate_programmes}++ if ($tv_guide->{$channel}->{data}->{$event_id});
[2]	340
	341	# wrap these non-mandatory fields in an eval so if they don't exist the script doesn't barf out
[73]	342	my %e;
	343	foreach my $field ('title', 'subtitle', 'description_1', 'main_cast', 'year_released', 'rating',
	344	'genre', 'running_time', 'repeat', 'country', 'movie', 'premiere', 'final', 'captions', 'warnings',
	345	'color', 'language', 'director', 'live') {
	346	eval { $e{$field} = $event->getElementsByTagName("$field")->item(0)->getFirstChild->getNodeValue; };
	347	}
[2]	348	# other fields we dont pick up but exist in source xml data include:
[73]	349	# other_title, description_2, genre_id, highlight, ext_url, y7_url
[2]	350
[73]	351	my @categories;
	352	push(@categories,"movie") if (($e{movie}) && ($e{movie} == 1));
	353	push(@categories,"premiere") if (($e{premiere}) && ($e{premiere} == 1));
	354	push(@categories,"final") if (($e{final}) && ($e{final} == 1));
	355	push(@categories,"live") if (($e{live}) && ($e{live} == 1));
	356	push(@categories,translate_category($e{genre})) if (($e{genre}) && ($e{genre} ne ""));
[2]	357
[73]	358	my %video_details;
	359	$video_details{'colour'} = "yes" if $e{color};
	360
	361	my @ratings;
	362	push(@ratings, [$e{rating}, 'ABA', undef]) if $e{rating};
	363	push(@ratings, [$e{warnings}, 'Warnings', undef]) if $e{warnings};
	364
	365	# store it in the correct XMLTV schema!
[2]	366	$tv_guide->{$channel}->{data}->{$event_id}->{'channel'} = $channels->{$channel};
[73]	367	$tv_guide->{$channel}->{data}->{$event_id}->{'start'} = strftime "%Y%m%d%H%M", localtime($event_start-$time_offset);
	368	$tv_guide->{$channel}->{data}->{$event_id}->{'stop'} = strftime "%Y%m%d%H%M", localtime($event_end-$time_offset);
	369	$tv_guide->{$channel}->{data}->{$event_id}->{'title'} = [[ $e{title}, $lang ]] if $e{title};
	370	$tv_guide->{$channel}->{data}->{$event_id}->{'sub-title'} = [[ $e{subtitle}, $lang ]] if $e{subtitle};
	371	$tv_guide->{$channel}->{data}->{$event_id}->{'desc'} = [[ $e{description_1}, $lang ]] if $e{description_1};
	372	$tv_guide->{$channel}->{data}->{$event_id}->{'category'} = [[ @categories ]] if @categories;
	373	$tv_guide->{$channel}->{data}->{$event_id}->{'country'} = [[ $e{country}, $lang ]] if $e{country};
	374	$tv_guide->{$channel}->{data}->{$event_id}->{'premiere'} = [ 'premiere', $lang ] if $e{premiere};
	375	$tv_guide->{$channel}->{data}->{$event_id}->{'rating'} = [ @ratings ];
	376	$tv_guide->{$channel}->{data}->{$event_id}->{'credits'}{'actor'} = [ split(/, /, $e{main_cast}) ] if $e{main_cast};
	377	$tv_guide->{$channel}->{data}->{$event_id}->{'credits'}{'director'} = [ split(/, /, $e{director}) ] if $e{director};
	378	$tv_guide->{$channel}->{data}->{$event_id}->{'date'} = $e{year_released} if $e{year_released};
	379	$tv_guide->{$channel}->{data}->{$event_id}->{'previously-shown'} = { } if $e{repeat};
	380	$tv_guide->{$channel}->{data}->{$event_id}->{'subtitles'} = [ { 'type' => 'teletext' } ] if $e{captions};
	381	$tv_guide->{$channel}->{data}->{$event_id}->{'last-chance'} = [ 'final', $lang ] if $e{final};
	382	$tv_guide->{$channel}->{data}->{$event_id}->{'video'} = \%video_details;
	383	$tv_guide->{$channel}->{data}->{$event_id}->{'length'} = ($e{running_time} * 60) if $e{running_time};
	384	$tv_guide->{$channel}->{data}->{$event_id}->{'language'} = [ split(/, /, $e{language}) ] if $e{language};
[2]	385	}
	386	}
	387	$tree->dispose;
	388	}
	389
	390	######################################################################################################
[73]	391
	392	sub translate_category
	393	{
	394	my $genre = shift;
	395	my %translation = (
	396	'Sport' => 'sports',
	397	'Soap Opera' => 'Soap',
	398	'Science and Technology' => 'Science/Nature',
	399	'Real Life' => 'Reality',
	400	'Cartoon' => 'Animation',
	401	'Family' => 'Children',
	402	'Murder' => 'Crime' );
	403
	404	return $translation{$genre} if defined $translation{$genre};
	405	return $genre;
	406	}
	407
	408	######################################################################################################
[2]	409	# descend a structure and clean up various things, including stripping
	410	# leading/trailing spaces in strings, translations of html stuff etc
	411	# -- taken & modified from Michael 'Immir' Smith's excellent tv_grab_au
	412
	413	my %amp;
	414	BEGIN { %amp = ( nbsp => ' ', qw{ amp & lt < gt > apos ' quot " } ) }
	415
	416	sub cleanup {
	417	my $x = shift;
	418	if (ref $x eq "REF") { cleanup($_) }
	419	elsif (ref $x eq "HASH") { cleanup(\$_) for values %$x }
	420	elsif (ref $x eq "ARRAY") { cleanup(\$_) for @$x }
	421	elsif (defined $$x) {
	422	$$x =~ s/&(#(\d+)\|(.*?));/ $2 ? chr($2) : $amp{$3}\|\|' ' /eg;
	423	# $$x =~ s/[^\x20-\x7f]/ /g;
	424	$$x =~ s/(^\s+\|\s+$)//g;
	425	}
	426	}
	427
	428	######################################################################################################
	429
	430	sub write_data
	431	{
	432	my %writer_args = ( encoding => 'ISO-8859-1' );
	433	if ($opt_outputfile) {
	434	my $fh = new IO::File(">$opt_outputfile") or die "can't open $opt_outputfile: $!";
	435	$writer_args{OUTPUT} = $fh;
	436	}
	437
	438	my $writer = new XMLTV::Writer(%writer_args);
	439
	440	$writer->start
	441	( { 'source-info-url' => "about:blank",
	442	'source-info-name' => "$progname $version",
	443	'generator-info-name' => "$progname $version"} );
	444
	445	for my $channel (sort keys %{$channels}) {
	446	$writer->write_channel( {
	447	'display-name' => [[ $channel, $lang ]],
	448	'id' => $channels->{$channel}
	449	} );
	450	}
	451
	452	for my $channel (sort keys %{$channels}) {
	453	for my $event_id (sort {$a <=> $b} keys %{($tv_guide->{$channel}->{data})}) {
	454	my $show = $tv_guide->{$channel}->{data}->{$event_id};
	455	&cleanup($show);
	456	$writer->write_programme($show);
	457	}
	458	}
	459
	460	$writer->end();
	461	}
	462
	463	######################################################################################################

Note: See TracBrowser for help on using the browser.

Context Navigation

root/grabbers/yahoo7widget @ 73

Download in other formats: