Index: /status
===================================================================
--- /status (revision 516)
+++ /status (revision 517)
@@ -8,5 +8,5 @@
 grabber         jrobbo              0.06
 grabber         ninemsn             0.11
-grabber         yahoo7web           0.07
+grabber         yahoo7web           0.08
 grabber         ten_website         0.05
 reconciler      reconciler_mk2      0.19
Index: /grabbers/yahoo7web.conf
===================================================================
--- /grabbers/yahoo7web.conf (revision 450)
+++ /grabbers/yahoo7web.conf (revision 517)
@@ -2,15 +2,15 @@
             'max_reliable_days' => 7,
             'channels' => '',
-            'option_anon_socks' => '--anonsocks',
             'option_days_offset' => '--offset',
             'option_offset_eats_days' => 1,
             'regions' => '-184',
             'option_ready' => '--version',
-            'desc' => 'yahoo7web fetches guide data (slowly) from au.tv.yahoo.com',
+            'desc' => 'yahoo7web fetches guide data from au.tv.yahoo.com',
             'max_days' => 7,
             'category' => 1,
+	    'micrograbs' => 1,
             'quality' => 3,
             'cache' => 1,
-	    'max_runtime' => '240',
+	    'max_runtime' => '60',
             'option_days' => '--days'
           };
Index: /grabbers/yahoo7web
===================================================================
--- /grabbers/yahoo7web (revision 450)
+++ /grabbers/yahoo7web (revision 517)
@@ -8,5 +8,5 @@
 
 my $progname = "yahoo7web";
-my $version = "0.07";
+my $version = "0.08";
 
 use LWP::UserAgent;
@@ -25,6 +25,7 @@
 $| = 1;
 my $script_start_time = time;
+my %amp = ( nbsp => ' ', qw{ amp & lt < gt > apos ' quot " } );
 my %stats;
-my $channels, my $opt_channels;
+my $channels, my $opt_channels, my $gaps;
 my $data_cache;
 my $writer;
@@ -51,4 +52,5 @@
 	'timezone=s'	=> \$opt->{timezone},
 	'channels_file=s' => \$opt->{channels_file},
+	'gaps_file=s'	=> \$opt->{gaps_file},
 	'output=s'	=> \$opt->{outputfile},
 	'cache-file=s'	=> \$opt->{cache_file},
@@ -81,5 +83,6 @@
 #
 
-&log(sprintf "going to grab %d days%s of data into %s (%s%s%s%s%s)",
+&log(sprintf "going to %sgrab %d days%s of data into %s (%s%s%s%s%s)",
+	(defined $opt->{gaps_file} ? "micro-gap " : ""),
 	$opt->{days},
 	(defined $opt->{offset} ? " (skipping first $opt->{offset} days)" : ""),
@@ -99,4 +102,14 @@
 }
 
+# if just filling in microgaps, parse gaps
+if (defined $opt->{gaps_file}) {
+	if (-r $opt->{gaps_file}) {
+		local (@ARGV, $/) = ($opt->{gaps_file});
+		no warnings 'all'; eval <>; die "$@" if $@;
+	} else {
+		die "WARNING: gaps_file $opt->{gaps_file} could not be read: $!\n";
+	}
+}
+
 &read_cache unless (defined $opt->{no_cache});
 
@@ -104,5 +117,4 @@
 &setup_socks if (defined $opt->{anon_socks});
 
-&build_venue_map;
 &start_writing_xmltv;
 
@@ -141,4 +153,6 @@
 	--region=N		set region for where to collect data from (default: $opt->{region})
 	--channels_file=file	where to get channel data from
+	--gaps_file=file	micro-fetch gaps only
+
 EOF
 ;
@@ -262,5 +276,5 @@
 		$attempts++;
 
-		my $sleep_for = 600;
+		my $sleep_for = 30;
 		$sleep_for = 10 if (defined $opt->{anon_socks});
 
@@ -281,9 +295,9 @@
 	$stats{http_successful_requests}++;
 
-	if ((!$opt->{fast}) || (!defined $opt->{anon_socks})) {
-		my $sleeptimer = int(rand(6)) + 17;  # sleep anywhere from 17 to 23 seconds
-		$stats{slept_for} += $sleeptimer;
-		sleep $sleeptimer;
-	}
+#	if ((!$opt->{fast}) || (!defined $opt->{anon_socks})) {
+#		my $sleeptimer = int(rand(6));
+#		$stats{slept_for} += $sleeptimer;
+#		sleep $sleeptimer;
+#	}
 
 	if ($response->header('Content-Encoding') &&
@@ -300,5 +314,5 @@
 {
 	my ($entry) = @_;
-	printf "%s [%d] %s\n",$progname,time,$entry;
+	printf "%s\n",$entry;
 }
 
@@ -318,7 +332,4 @@
 # leading/trailing spaces in strings, translations of html stuff etc
 #   -- taken & modified from Michael 'Immir' Smith's excellent tv_grab_au
-
-my %amp;
-BEGIN { %amp = ( nbsp => ' ', qw{ amp & lt < gt > apos ' quot " } ) }
 
 sub cleanup {
@@ -408,8 +419,6 @@
 ##############################################################################
 
-sub build_venue_map
-{
-	&log("fetching initial channel-to-venue map");
-
+sub build_channel_quirks_map
+{
 	# set up channel name exceptions list
 	my %chan_map;
@@ -437,46 +446,5 @@
 	}
 
-
-	my $url = sprintf "http://au.tv.yahoo.com/results.html?vn=&rg=%d&dt=%s&ts=12&x=19&y=14",
-	  $opt->{region}, (strftime "%Y-%m-%d",localtime(time));
-
-	my $data = &get_url($url,5);
-	if (!$data) {
-		&log("CRITICAL ERROR: Could not build venue map because of error fetching '$url'");
-		exit(1);
-	}
-
-	my $tree = HTML::TreeBuilder->new_from_content($data);
-	if (!$tree) {
-		&log("CRITICAL ERROR: url '$url' doesn't seem to contain any valid HTML: has the format changed?");
-                exit(1);
-        }
-
-	foreach my $venuelink ($tree->look_down('_tag' => 'a')) {
-		my $venueurl = $venuelink->attr('href');
-		if (($venueurl) && ($venueurl =~ /^venueresults.html.*vn=(\d+)$/)) {
-			my $venue_id = $1;
-			my $channame = $venuelink->as_text();
-
-			if (($channame) && ($channame ne "")) {
-				# see if we have an alternate name for this channel
-				if (defined $chan_map{$channame}) {
-					my $new_channame = splice(@{($chan_map{$channame})},0,1);
-					printf "DEBUG: mapped channel '%s' to new channel name '%s'\n",
-					  $channame, $new_channame if (defined $opt->{debug});
-					$channame = $new_channame;
-				}
-
-				if (defined $channels->{$channame}) {
-					$d->{venuemap}->{$channame} = $venue_id;
-
-					printf "DEBUG: mapped channel '%s' to venue '%s'\n",
-					  $channame, $venue_id if (defined $opt->{debug});
-				} else {
-					printf "DEBUG: ignored unknown channel '%s'\n", $channame if (defined $opt->{debug});
-				}
-			}
-		}
-	}
+	return %chan_map;
 }
 
@@ -488,8 +456,15 @@
 	my $day_num = 0;
 	my $skip_days = 0;
+	$stats{programmes} = 0;
+
+	my @timeattr = localtime($starttime); # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
+	$timeattr[0] = 0;	# zero sec
+	$timeattr[1] = 0;	# zero min
+	$timeattr[2] = 0;	# zero hour (midnight)
+	my $starttime_midnight = mktime(@timeattr);
 
 	$skip_days = $opt->{offset} if (defined $opt->{offset});
 	while ($day_num < $opt->{days}) {
-		my $currtime = $starttime + (60*60*24 * $day_num);
+		my $day_start = $starttime_midnight + (60*60*24 * $day_num);
 		$day_num++;
 
@@ -499,21 +474,29 @@
 			next;
 		}
-
-		my @timeattr = localtime($currtime); # 0=sec,1=min,2=hour,3=day,4=month,5=year,6=wday,7=yday,8=isdst
-		$timeattr[0] = 0; # zero sec
-		$timeattr[1] = 0; # zero min
-		$timeattr[2] = 0; # zero hour
-		my $day_start = mktime(@timeattr); # midnight on the day
-
-		foreach my $ch (keys %$channels) {
-			if ((!defined $d->{venuemap}->{$ch}) && (!defined $d->{skipmap}->{$ch})) {
-				&log("channel '$ch' skipped due to no channel-to-vanue mapping data");
-				$d->{skipmap}->{$ch} = 1;	# report this only once
-				next;
-			}
-
-			&log("fetching day $day_num summary page for '$ch'");
-			&parse_summary_page($day_start, $d->{venuemap}->{$ch}, $channels->{$ch}, $day_num);
-		}
+		
+		# within each day, fetch in groups of 3 hours
+		for (my $hr = 0; $hr < 23; $hr += 3) {
+			my $currtime = $day_start + ($hr * 60 * 60);
+			next if (($currtime + (3 * 60 * 60)) < $starttime); # no point fetching the past
+
+			my $url = sprintf "http://au.tv.yahoo.com/tv-guide/?hour=%s&min=%s&date=%s&mon=%s&year=%s&tvrg=%s&next=%s",
+				POSIX::strftime("%H",localtime($starttime)),
+				POSIX::strftime("%M",localtime($starttime)),
+				POSIX::strftime("%d",localtime($starttime)),
+				POSIX::strftime("%m",localtime($starttime)),
+				POSIX::strftime("%Y",localtime($starttime)),
+				$opt->{region}, $currtime;
+
+			&log("fetching day $day_num summary page hour $hr ($url)");
+			&parse_summary_page($url, $day_num, $day_start);
+
+			my $wait_for = 2;
+			$stats{slept_for} += $wait_for;
+			sleep($wait_for);
+		}
+
+		my $wait_for = 5 + int(rand(5));
+		$stats{slept_for} += $wait_for;
+		sleep($wait_for);
 	}
 }
@@ -523,8 +506,7 @@
 sub parse_summary_page
 {
-	my ($day_start, $venueid, $xmlid, $day_num) = @_;
-
-	my $url = sprintf "http://au.tv.yahoo.com/venueresults.html?dt=%s&vn=%d",
-		(strftime "%Y-%m-%d",localtime($day_start)), $venueid;
+	my ($url, $day_num, $day_start) = @_;
+	my %chan_map = &build_channel_quirks_map;
+
 	my $data = &get_url($url,5);
 	return if (!$data);
@@ -536,5 +518,5 @@
 	}
 
-	my $tree_table = $tree->look_down('_tag' => 'table', 'width' => '100%', 'border' => '1', 'bordercolor' => '#efefef');
+	my $tree_table = $tree->look_down('_tag' => 'table', 'id' => 'listing-table');
 	if (!$tree_table) {
 		&log("url '$url' doesn't seem to contain a TV table.  Has the format changed?");
@@ -542,104 +524,101 @@
 	}
 
-	$stats{programmes} = 0 if (!defined $stats{programmes});
-	my $progs_in_day = 0;
-	my $rownum = 0;
-	my $seen_am = 0;
-	my $seen_pm = 0;
-
-	for my $tree_tr ($tree_table->look_down('_tag' => 'tr')) {
-		$rownum++;
-		next if ($rownum == 1); # skip header
-
-		my @tree_col = $tree_tr->look_down('_tag' => 'td');
-
-		if ($#tree_col != 1) {
-			&log("WARNING: unexpected number of columns in table from '$url' in table row $rownum; ignoring: URL format changed?");
+	my $progs_in_table = 0;
+
+	for my $tree_tr ($tree_table->look_down('_tag' => 'tr', 'class' => 'lt-listing-row')) {
+		# get channel
+		my $this_chan = "";
+		if (my $channel_td = $tree_tr->look_down('_tag' => 'td', 'class' => 'lt-channel')) {
+			$this_chan = $channel_td->as_text();
+		}
+
+		if ($this_chan eq "") {
+			&log("ignoring blank channel in $url") if (defined $opt->{debug});
+			$stats{blank_channels_ignored}++;
 			next;
 		}
 
-		my $prog;
-		$prog->{channel} = $xmlid;
-
-		# first column contains time
-		if ($tree_col[0]->as_text() =~ /^(\d+):(\d+)(.)M$/) {
-			my $hr = $1;
-			my $min = $2;
-			my $ampm = lc($3);
-
-			$hr = 0 if ($hr == 12);		# convert to 24hr format
-			$hr += 12 if ($ampm eq "p");
-
-			$hr -= 24 if (($seen_am == 0) && ($ampm eq "p")); # yesterday
-			$hr += 24 if (($seen_pm == 1) && ($ampm eq "a")); # tomorrow
-
-			$prog->{starttime} = $day_start + ((60*60)*$hr) + (60*$min);
-			printf "DEBUG: Parsed time '%s' on day %d to be %s\n",
-			  $tree_col[0]->as_text(), $day_num, (strftime "%Y%m%d%H%M",
-			  localtime($prog->{starttime})) if (defined $opt->{debug});
-
-			$seen_am = 1 if ($ampm eq "a");
-			$seen_pm = 1 if (($ampm eq "p") && ($seen_am));
-		} else {
-			&log((sprintf "Couldn't parse malformed time '%s' in row $rownum of '%s': ignored.",
-			  $tree_col[0]->as_text(), $url));
-			$stats{malformed_time}++;
+		if (defined $chan_map{$this_chan}) {
+			my $new_channame = splice(@{($chan_map{$this_chan})},0,1);
+			&log("substituted channel name '$new_channame' for '$this_chan'") if (defined $opt->{debug});
+			$stats{substituted_channels}++;
+			$this_chan = $new_channame;
+		}
+
+		if (!defined $channels->{$this_chan}) {
+			&log("skipping unlisted channel '$this_chan'") if (!defined $d->{skipped_channels}->{$this_chan});
+			$d->{skipped_channels}->{$this_chan} = 1 if (!defined $opt->{debug});
+			$stats{skipped_channels}++;
 			next;
 		}
 
-		# second column contains programme title + rating
-		if (my $detail_link = $tree_col[1]->look_down('_tag' => 'a')) {
-			$prog->{title} = [[ $detail_link->as_text(), $opt->{lang} ]];
-
-			my $progurl = $detail_link->attr('href');
-			$progurl =~ s#^javascript:popup\(\"(.*)\"\)#http://au.tv.yahoo.com/$1#g;
-
-			if (defined $d->{already_seen_details}->{$progurl}) {
-				printf "DEBUG: skipping prog '%s' as we have seen its URL before '%s'\n",
-				  $prog->{title}->[0]->[0], $progurl if (defined $opt->{debug});
-				$stats{duplicate_programme}++;
-				next;
+		for my $tree_td ($tree_tr->look_down('_tag' => 'td', 'class' => 'lt-listing')) {
+			if (my $listing_div = $tree_td->look_down('_tag' => 'div')) {
+				next if ($listing_div->attr('class') !~ /^lt-listing-wrapper/i);
+
+				my @listing_links = $listing_div->look_down('_tag' => 'a', 'class' => 'listing-link');
+				my @listing_data = $listing_div->look_down('_tag' => 'strong');
+
+				for (my $i=0; $i <= $#listing_links; $i++) {
+					my $prog;
+					$prog->{channel} = $channels->{$this_chan};
+
+					if ($listing_links[$i]->attr('rel') =~ /^(\d+)-(\d+)-(\d+)$/) {
+						$prog->{event_id} = $3;
+					}
+					$prog->{title} = [[ $listing_links[$i]->as_text(), $opt->{lang} ]];
+
+					my $listing_text = $listing_data[$i]->as_text();
+					if ($listing_text =~ /^(.*)\((\d+)\)(\d+):(\d+)(.)m - (\d+):(\d+)(.)m$/i) {
+						my ($rating_text, $prog_length, $start_sec, $stop_sec) = ($1, $2, parse_time($3, $4, $5), parse_time($6, $7, $8));
+
+						$prog->{rating} = [[ $rating_text, 'ABA', undef ]] if ((defined $rating_text) && ($rating_text ne ""));
+						$prog->{length} = ($prog_length * 60) if ((defined $prog_length) && ($prog_length > 0));
+						$prog->{starttime} = $day_start + $start_sec;
+						$prog->{stoptime} = $day_start + $stop_sec;
+					} else {
+						&log("malformed listing_text '$listing_text' for prog '".$listing_links[$i]->as_text()."'; ignored.");
+						$stats{malformed_listing}++;
+						next;
+					}
+
+					$progs_in_table++;
+
+					# if we are fetching microgaps, skip if this isn't
+					# in a micro-gap.
+					if (defined $opt->{gaps_file}) {
+						my $found_gap_match = 0;
+						if (defined $gaps->{$prog->{channel}}) {
+							foreach my $g (@{($gaps->{$prog->{channel}})}) {
+								my ($s, $e) = split(/-/,$g);
+								$found_gap_match = 1 if
+								  ((($s >= $prog->{starttime}) && ($s <= $prog->{stoptime})) ||
+								   (($e >= $prog->{starttime}) && ($e <= $prog->{stoptime})) ||
+								   (($s <= $prog->{starttime}) && ($e >= $prog->{stoptime})));
+							}
+						}
+						if (!$found_gap_match) {
+							$stats{gaps_skipped}++;
+							next;
+						} else {
+							$stats{gaps_included}++;
+						}
+					}
+
+					# include programme
+					&log("found prog: '".$prog->{title}->[0]->[0]."', channel ".$prog->{channel}.
+					  " start ".$prog->{starttime}." stop ".$prog->{stoptime}) if (defined $opt->{debug});
+
+					my $cache_key = sprintf "%d:%s:%s", $prog->{starttime}, $prog->{channel}, $prog->{title}->[0]->[0];
+					if (!defined $d->{progs}->{$cache_key}) {
+						$d->{progs}->{$cache_key} = $prog;
+						$stats{programmes}++;
+					}
+				}
 			}
-
-			$d->{already_seen_details}->{$progurl} = 1;
-			$prog->{url} = $progurl;
-		} else {
-			&log((sprintf "couldn't find a programme name/url for programme at %s in row %d of '%s': ignored.",
-			  (strftime "%Y%m%d%H%M",localtime($prog->{starttime})), $rownum, $url));
-			$stats{prog_with_no_name}++;
-			next;
-		}
-
-		# see if we can derive an ABA rating
-		if ($tree_col[1]->as_text() =~ / - ([A-Z]+)$/) {
-			my @ratings;
-			push(@ratings, [$1, 'ABA', undef]);
-			$prog->{rating} = [ @ratings ];
-
-			printf "DEBUG: prog '%s' has a rating of '%s'\n",
-			  $prog->{title}->[0]->[0], $1 if (defined $opt->{debug});
-		}
-
-		push(@{($d->{progs})},$prog);
-
-		printf "DEBUG: added prog %d from row %d: %s '%s' on channel '%s'\n", $stats{programmes},
-		  $rownum, (strftime "%Y%m%d%H%M",localtime($prog->{starttime})), $prog->{title}->[0]->[0], 
-		  $prog->{channel} if (defined $opt->{debug});
-
-		# if we can fill in a stoptime for previous program on this channel on this day, do so!
-		if ($progs_in_day > 0) {
-			$d->{progs}->[($stats{programmes})]->{stoptime} = $prog->{starttime};
-
-			printf "DEBUG: added stoptime of '%s' to previous programme %d\n",
-			  (strftime "%Y%m%d%H%M",localtime($prog->{starttime})), ($stats{programmes}-1)
-			  if (defined $opt->{debug});
-		}
-
-		$stats{programmes}++;
-		$progs_in_day++;
-	}
-
-	&log("WARNING: $progs_in_day programmes seen on day $day_num for channel '$xmlid' in '$url'. ".
-	  "Data may be bad.") if ($progs_in_day < 10);
+		}
+	}
+
+	&log("WARNING: Data may be bad. Only $progs_in_table programmes seen in $url") if ($progs_in_table < 5);
 }
 
@@ -651,24 +630,27 @@
 sub get_detailed_pages
 {
+	&log("fetching details for up to ".$stats{programmes}." programmes ...") if (!defined $opt->{no_details});
+
 	my $prog_count = 0;
 	my $added_to_cache = 0;
-
-	foreach my $prog (@{($d->{progs})}) {
+	$stats{used_existing_cache_entry} = 0;
+	$stats{added_to_cache} = 0;
+
+	foreach my $cache_key (sort keys %{($d->{progs})}) {
+		my $prog = $d->{progs}->{$cache_key};
 		$prog_count++;
-		my $cache_key = sprintf "%d:%s:%s", $prog->{starttime}, $prog->{channel}, $prog->{title}->[0]->[0];
-
-		if ((!defined $data_cache->{$cache_key}) && (!defined $opt->{no_details}) &&
+
+		if ((!defined $data_cache->{$cache_key}) &&
+		    (!defined $opt->{no_details}) &&
+		    (defined $prog->{event_id}) &&
 		    ($prog->{title}->[0]->[0] ne "Station Close")) {
-			printf "DEBUG: Fetching detail page: %s: %s\n",
-			  $prog->{channel}, $prog->{url} if (defined $opt->{debug});
-
-			# not in cache, go fetch additional details if we can
-			&log("fetching programme detail page ($prog_count of $stats{programmes})");
-			&fetch_one_prog($cache_key, $prog->{url});
-
-			$stats{added_to_cache}++;
-			&write_cache if ((($stats{added_to_cache} % 5) == 0) && (!defined $opt->{no_cache}));
+			&fetch_one_prog($cache_key, $prog->{event_id});
+			&write_cache if ((($stats{added_to_cache} % 15) == 0) && (!defined $opt->{no_cache}));
 		} elsif (!defined $opt->{no_details}) {
 			$stats{used_existing_cache_entry}++;
+		}
+
+		if ((($prog_count % 25) == 0) && (!defined $opt->{no_details})) {
+			&log(" ... at ".$prog_count." of ".$stats{programmes}." programmes (used ".$stats{used_existing_cache_entry}." from cache)");
 		}
 
@@ -680,27 +662,16 @@
 		}
 
-		# if we now have a length field, use that as a more accurate
-		# stop time (we may have got a length field in the detailed data)
-		$prog->{stoptime} = $prog->{starttime} + (60*$prog->{length})
-		  if (defined $prog->{length});
-
-		# if we don't have a stoptime, skip prog
-		if (!defined $prog->{stoptime}) {
-			$stats{skipped_no_stoptime}++;
-			next;
-		}
-
 		# convert epoch starttime into XMLTV starttime
-		$prog->{start} = strftime "%Y%m%d%H%M", localtime($prog->{starttime});
+		$prog->{start} = POSIX::strftime("%Y%m%d%H%M", localtime($prog->{starttime}));
 		delete $prog->{starttime};
 
 		# convert epoch stoptime into XMLTV stoptime
-		$prog->{stop} = strftime "%Y%m%d%H%M", localtime($prog->{stoptime});
+		$prog->{stop} = POSIX::strftime("%Y%m%d%H%M", localtime($prog->{stoptime}));
 		delete $prog->{stoptime};
 
-		delete $prog->{url};
+		delete $prog->{event_id};
 		&cleanup($prog);
 
-		printf "DEBUG: programme xmltv: ".Dumper($prog) if (defined $opt->{debug});
+		printf "DEBUG: programme xmltv: ".Dumper($prog) if ((defined $opt->{debug}) && ($opt->{debug} > 1));
 		$writer->write_programme($prog);
 	}
@@ -711,65 +682,78 @@
 sub fetch_one_prog
 {
-	my ($cache_key, $url) = @_;
-
+	my ($cache_key, $event_id) = @_;
+	&log("fetching detail page for $cache_key with event_id $event_id") if (defined $opt->{debug});
+
+	my $url = "http://au.tv.yahoo.com/tv-guide/broker.html?event_id=".$event_id;
 	my $data = &get_url($url,5);
-	return if (!$data);
-
-	my $tree = HTML::TreeBuilder->new_from_content($data);
-	return if (!$tree);
-
-	my @categories;
-	my $desc = "";
-
-	if (my $inner_tree = $tree->look_down('_tag' => 'div', 'class' => 'inner')) {
-		if ($_ = $inner_tree->look_down('_tag' => 'h1')) {
-			$data_cache->{$cache_key}->{title} = [[ $_->as_text(), $opt->{lang} ]] if ($_->as_text() ne "");
-		}
-
-		if ($_ = $inner_tree->look_down('_tag' => 'h2')) {
-			$data_cache->{$cache_key}->{'sub-title'} = [[ $_->as_text(), $opt->{lang} ]] if ($_->as_text() ne "");
-		}
-
-		foreach my $para ($inner_tree->look_down('_tag' => 'p')) {
-			if ($para->as_HTML() =~ /<p>Genre:&nbsp; (.*)$/) {
-				push(@categories,translate_category($1), $opt->{lang});
-			} elsif ($para->as_HTML() =~ /(\d+)&nbsp;mins/) {
-				$data_cache->{$cache_key}->{'length'} = ($1*60);
-
-				# any other tags in here?
-				if ($para->as_HTML() =~ /Premiere/) {
-					$data_cache->{$cache_key}->{premiere} = [ 'premiere', $opt->{lang} ];
-					push(@categories,"premiere",$opt->{lang});
-				}
-
-				if ($para->as_HTML() =~ /Final/) {
-					$data_cache->{$cache_key}->{'last-chance'} = [ 'final', $opt->{lang} ];
-					push(@categories,"final",$opt->{lang});
-				}
-
-				push(@categories,"highlight",$opt->{lang})
-				  if ($para->as_HTML() =~ /highlight/);
-
-				push(@categories,"live",$opt->{lang})
-				  if ($para->as_HTML() =~ /Live/);
-
-				$data_cache->{$cache_key}->{'previously-shown'} = { }
-				  if ($para->as_HTML() =~ /Repeat/);
-
-				$data_cache->{$cache_key}->{'subtitles'} = [ { 'type' => 'teletext' } ]
-				  if ($para->as_HTML() =~ /Closed Captions/);
+
+	if ((!$data) || ($data !~ /^\{.*\}$/)) {
+		$stats{bad_details_page}++;
+		return;
+	}
+
+	$stats{added_to_cache}++;
+
+	if (($stats{added_to_cache} % 35) == 0) {
+		my $wait_for = 12 + int(rand(5));
+		$stats{slept_for} += $wait_for;
+		sleep $wait_for;
+	}
+
+	my @genre;
+
+	$data =~ s/(^\{|\}$)//g; # strip leading/trailing { and }
+	foreach my $field_item (split(/,"/,$data)) {
+		if ($field_item =~ /^([A-Za-z0-9\_\"]+):(.*)/) {
+			my ($f, $v) = ($1, $2);
+			$f =~ s/(^\"|\"$)//g;	# strip leading/trailing quotes from field if present
+			$v =~ s/(^\"|\"$)//g;	# strip leading/trailing quotes from value if present
+			next if ($v eq "");
+
+			if ($f eq "title") {
+				; # nothing
+			} elsif ($f eq "subtitle") {
+				$data_cache->{$cache_key}->{'sub-title'} = [[ $v, $opt->{lang} ]];
+			} elsif ($f eq "description") {
+				$data_cache->{$cache_key}->{desc} = [[ $v, $opt->{lang} ]];
+			} elsif ($f eq "genre") {
+				push(@genre, translate_category($v));
+			} elsif ($f eq "captions") {
+				$data_cache->{$cache_key}->{subtitles} = [ { 'type' => 'teletext' } ] if ($v eq "true");
+			} elsif ($f eq "start_date") {
+				; # nothing
+			} elsif ($f eq "end_date") {
+				; # nothing
+			} elsif ($f eq "rating") {
+				; # nothing
+			} elsif ($f eq "channel") {
+				; # nothing
+			} elsif ($f eq "hotpick") {
+				; # nothing
+			} elsif ($f eq "venue_url") {
+				; # nothing
+			} elsif ($f eq "url") {
+				; # nothing
+			} elsif ($f eq "alt_url") {
+				; # nothing
+			} elsif ($f eq "alt_text") {
+				; # nothing
+			} elsif ($f eq "img") {
+				; # nothing
 			} else {
-				$desc .= $para->as_text();
+				&log("unknown field '$f' in $url") if (!defined $d->{unknown_fields}->{$f});
+				$d->{unknown_fields}->{$f} = 1;
 			}
-		}
-
-		$data_cache->{$cache_key}->{desc} = [[ $desc, $opt->{lang} ]] if ($desc ne "");
-		$data_cache->{$cache_key}->{category} = [[ @categories ]] if @categories;
-
-		&cleanup($data_cache->{$cache_key});
-
-		printf "DEBUG: cached entries for '$cache_key': ".Dumper($data_cache->{$cache_key})
-		  if (defined $opt->{debug});
-	}
+
+			$data_cache->{$cache_key}->{category} = [[ @genre ]] if ($#genre != -1);
+
+		} else {
+			&log("unknown field format '$field_item' in details. Has the format changed?");
+			$stats{unknown_details_field_format}++;
+		}
+	}
+
+	printf "DEBUG: cached entries for '$cache_key': ".Dumper($data_cache->{$cache_key})
+	  if (defined $opt->{debug});
 }
 
@@ -857,2 +841,15 @@
 
 ##############################################################################
+
+sub parse_time
+{
+	my ($hr, $min, $ampm) = @_;
+
+	$hr = 0 if ($hr == 12);
+	$hr += 12 if ($ampm =~ /p/i);
+
+	return(($hr*60*60)+($min*60));
+}
+
+##############################################################################
+