| 1 | #!/usr/bin/perl |
|---|
| 2 | |
|---|
| 3 | # flag_aus_hdtv |
|---|
| 4 | # checks the digital broadcasting australia (www.dba.org.au) site for |
|---|
| 5 | # their condensed program guide information for information such as |
|---|
| 6 | # widescreen, "high definition" (notwithstanding channel seven's |
|---|
| 7 | # definition as 576p), DD5.1 and updates supplied xmltv |
|---|
| 8 | # to map SD to SD+HD channels |
|---|
| 9 | # - Richard Dale <richard@interlink.com.au>, September 2006 |
|---|
| 10 | |
|---|
| 11 | my $progname = "flag_aus_hdtv"; |
|---|
| 12 | my $version = "0.10.2"; |
|---|
| 13 | $| = 1; |
|---|
| 14 | printf "%s v%s\n",$progname,$version; |
|---|
| 15 | |
|---|
| 16 | use strict; |
|---|
| 17 | use Getopt::Long; |
|---|
| 18 | use LWP::UserAgent; |
|---|
| 19 | use HTML::TreeBuilder; |
|---|
| 20 | use IO::File; |
|---|
| 21 | use XMLTV; |
|---|
| 22 | use Data::Dumper; |
|---|
| 23 | |
|---|
| 24 | # |
|---|
| 25 | # customizations |
|---|
| 26 | # |
|---|
| 27 | |
|---|
| 28 | # Add overrides for HD - sometimes DBA doesn't always have the latest HD information about TV shows so set them here in the override |
|---|
| 29 | my $hdoverride; |
|---|
| 30 | $hdoverride->{TEN} = [ "The Handler", "N.Y.P.D. Blue", "The Office", "Battlestar Galactica" ]; |
|---|
| 31 | |
|---|
| 32 | # Add translations for show names that differ between DBA and our normal tv_grab_au names |
|---|
| 33 | my $translation; |
|---|
| 34 | $translation->{'ACA'} = 'A Current Affair'; |
|---|
| 35 | $translation->{'CSI'} = 'CSI: Crime Scene Investigation'; |
|---|
| 36 | $translation->{'CSI-Miami'} = 'CSI: Miami'; |
|---|
| 37 | |
|---|
| 38 | my $chan_name_translations; |
|---|
| 39 | $chan_name_translations->{"Ten"} = "TEN"; |
|---|
| 40 | $chan_name_translations->{"ABC Main"} = "ABC"; |
|---|
| 41 | |
|---|
| 42 | # |
|---|
| 43 | # options |
|---|
| 44 | # |
|---|
| 45 | |
|---|
| 46 | my $ua; |
|---|
| 47 | $ua = LWP::UserAgent->new('timeout' => 30, 'keep_alive' => 1, 'agent' => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-us)' ); |
|---|
| 48 | $ua->env_proxy; |
|---|
| 49 | my %stats; |
|---|
| 50 | my $channels, my $opt_channels; |
|---|
| 51 | my $hdprog; |
|---|
| 52 | my %amp = ( nbsp => ' ', qw{ amp & lt < gt > apos ' quot " } ); |
|---|
| 53 | |
|---|
| 54 | my $opt; |
|---|
| 55 | $opt->{dbaurl} = "http://www.dba.org.au/dtvprograms"; # This URL seems to work fine Sep 2006 |
|---|
| 56 | $opt->{output_file} = "output.xmltv"; |
|---|
| 57 | $opt->{region} = 94; |
|---|
| 58 | $opt->{debug} = 1; |
|---|
| 59 | |
|---|
| 60 | &parse_command_line; |
|---|
| 61 | exit 0 if ($opt->{version}); |
|---|
| 62 | &show_help if ($opt->{help}); |
|---|
| 63 | die "'--channels_file {file}' must be specified. see --help for details.\n" if (!$opt->{channels_file}); |
|---|
| 64 | die "'--dontflagalldays' not yet implemented. sorry!\n" if ($opt->{dontflagalldays}); |
|---|
| 65 | |
|---|
| 66 | &read_config_file($opt->{channels_file},1); |
|---|
| 67 | |
|---|
| 68 | $opt->{locality} = decode_region($opt->{region}); |
|---|
| 69 | die "no HDTV data available for region $opt->{region}" if (!$opt->{locality}); |
|---|
| 70 | |
|---|
| 71 | my $data = &get_url($opt->{dbaurl},"Obtaining DBA data",0); |
|---|
| 72 | &parse_data($data) if ($data); |
|---|
| 73 | |
|---|
| 74 | # |
|---|
| 75 | # now eat up the XMLTV inputs we were given! |
|---|
| 76 | # |
|---|
| 77 | |
|---|
| 78 | @ARGV = ('-') if not @ARGV; |
|---|
| 79 | |
|---|
| 80 | my %writer_args = ( encoding => 'ISO-8859-1' ); |
|---|
| 81 | my $fh = new IO::File(">".$opt->{output_file}) || |
|---|
| 82 | die "can't open $opt->{output_file} for writing: $!"; |
|---|
| 83 | $writer_args{OUTPUT} = $fh; |
|---|
| 84 | |
|---|
| 85 | my $writer = new XMLTV::Writer(%writer_args); |
|---|
| 86 | $writer->start( { |
|---|
| 87 | 'source-info-name' => "$progname $version", |
|---|
| 88 | 'generator-info-name' => "$progname $version"} ); |
|---|
| 89 | |
|---|
| 90 | foreach my $file (@ARGV) { |
|---|
| 91 | printf "Parsing: %s\n",($file eq "-" ? "(from-stdin, hit control-D to finiah)" : $file); |
|---|
| 92 | XMLTV::parsefiles_callback(undef, undef, \&channel_cb,\&programme_cb, $file); |
|---|
| 93 | } |
|---|
| 94 | |
|---|
| 95 | $writer->end(); |
|---|
| 96 | printf "Finished parsing, output in $opt->{output_file}\n"; |
|---|
| 97 | |
|---|
| 98 | &print_stats; |
|---|
| 99 | exit(0); |
|---|
| 100 | |
|---|
| 101 | ###################################################################################################### |
|---|
| 102 | # read settings |
|---|
| 103 | |
|---|
| 104 | sub read_config_file |
|---|
| 105 | { |
|---|
| 106 | my($file,$die_on_failure) = @_; |
|---|
| 107 | if (!(-r $file)) { |
|---|
| 108 | die "file $file could not be read. aborting.\n" if $die_on_failure; |
|---|
| 109 | return; |
|---|
| 110 | } |
|---|
| 111 | local (@ARGV, $/) = ($file); |
|---|
| 112 | no warnings 'all'; eval <>; die "$@" if $@; |
|---|
| 113 | } |
|---|
| 114 | |
|---|
| 115 | ###################################################################################################### |
|---|
| 116 | |
|---|
| 117 | sub parse_command_line |
|---|
| 118 | { |
|---|
| 119 | GetOptions( |
|---|
| 120 | 'channels_file=s' => \$opt->{channels_file}, |
|---|
| 121 | 'config=s' => \$opt->{config_file}, |
|---|
| 122 | 'region=i' => \$opt->{region}, |
|---|
| 123 | 'dbaurl=s' => \$opt->{dbaurl}, |
|---|
| 124 | 'output=s' => \$opt->{output_file}, |
|---|
| 125 | 'dontflagalldays' => \$opt->{dontflagalldays}, |
|---|
| 126 | 'debug+' => \$opt->{debug}, |
|---|
| 127 | 'help' => \$opt->{help}, |
|---|
| 128 | 'h' => \$opt->{help}, |
|---|
| 129 | 'v' => \$opt->{version}, |
|---|
| 130 | 'version' => \$opt->{version}); |
|---|
| 131 | } |
|---|
| 132 | |
|---|
| 133 | ###################################################################################################### |
|---|
| 134 | |
|---|
| 135 | sub show_help |
|---|
| 136 | { |
|---|
| 137 | print<<EOF |
|---|
| 138 | $progname $version |
|---|
| 139 | |
|---|
| 140 | options: |
|---|
| 141 | --channels_file=(f) shepherd channels file (mandatory) |
|---|
| 142 | --region=(region) region where we are (mandatory) (default: $opt->{region}) |
|---|
| 143 | (VIC) 90,93,94,95,98 |
|---|
| 144 | (NSW/ACT) 63,66,67,69,71,73,106,184 |
|---|
| 145 | (QLD) 75,78,79,114 |
|---|
| 146 | (SA) 81,82,83,85,86,107 |
|---|
| 147 | (WA) 101,102 |
|---|
| 148 | (no HDTV in TAS or NT apparently) |
|---|
| 149 | --dbaurl=(url) URL where dba lists HDTV programming (default: $opt->{dbaurl}) |
|---|
| 150 | |
|---|
| 151 | --dontflagalldays don't flag a programme as being in HD across all days |
|---|
| 152 | if we see it in HD on one day. (dba website is often |
|---|
| 153 | incorrect with regard to what day a HD programme is on) |
|---|
| 154 | (default: do) |
|---|
| 155 | |
|---|
| 156 | --output=(f) output XMLTV filename (default: $opt->{output}) |
|---|
| 157 | |
|---|
| 158 | --debug increase debug level |
|---|
| 159 | |
|---|
| 160 | EOF |
|---|
| 161 | ; |
|---|
| 162 | exit(0); |
|---|
| 163 | } |
|---|
| 164 | |
|---|
| 165 | ###################################################################################################### |
|---|
| 166 | |
|---|
| 167 | sub decode_region |
|---|
| 168 | { |
|---|
| 169 | my $r = shift; |
|---|
| 170 | |
|---|
| 171 | return "M" if (($r==90)||($r==93)||($r==94)||($r==95)||($r==98)); |
|---|
| 172 | return "W" if (($r==101)||($r==102)); |
|---|
| 173 | return "S" if (($r==63)||($r==66)||($r==67)||($r==69)||($r==71)||($r==73)||($r==76)||($r==106)||($r==184)); |
|---|
| 174 | return "B" if (($r==75)||($r==78)||($r==79)||($r==114)); |
|---|
| 175 | return "A" if (($r==81)||($r==82)||($r==83)||($r==85)||($r==86)||($r==107)); |
|---|
| 176 | return undef; |
|---|
| 177 | } |
|---|
| 178 | |
|---|
| 179 | ###################################################################################################### |
|---|
| 180 | |
|---|
| 181 | sub parse_data |
|---|
| 182 | { |
|---|
| 183 | my $tree = HTML::TreeBuilder->new_from_content($data); |
|---|
| 184 | |
|---|
| 185 | # each daily section starts with: |
|---|
| 186 | # <TABLE class=tv... |
|---|
| 187 | # so we'll search for that and populate an array |
|---|
| 188 | my @chan_map; |
|---|
| 189 | for ($tree->look_down('_tag' => 'table', 'class' => 'tv')) { |
|---|
| 190 | my $firstrowdone=0; |
|---|
| 191 | my $dayname = ""; |
|---|
| 192 | |
|---|
| 193 | foreach my $row ($_->look_down('_tag' => 'tr')) { |
|---|
| 194 | if ($firstrowdone == 0) { # Channel list is the first row |
|---|
| 195 | $firstrowdone = 1; |
|---|
| 196 | my $colnum = 0; |
|---|
| 197 | for my $channel ($row->look_down('_tag' => 'td')) { |
|---|
| 198 | my $this_chan = $channel->as_text; |
|---|
| 199 | $colnum++; |
|---|
| 200 | $this_chan =~ s/[^\x20-\x7f]/ /g; |
|---|
| 201 | $this_chan =~ s/(^\s+|\s+$)//g; |
|---|
| 202 | next if ($this_chan eq ""); |
|---|
| 203 | |
|---|
| 204 | $this_chan = $chan_name_translations->{$this_chan} |
|---|
| 205 | if (defined $chan_name_translations->{$this_chan}); |
|---|
| 206 | |
|---|
| 207 | if (!defined $channels->{$this_chan}) { |
|---|
| 208 | printf "DBA listed unkown channel '$this_chan'! Ignored.\n"; |
|---|
| 209 | $stats{unknown_channels}++; |
|---|
| 210 | next; |
|---|
| 211 | } |
|---|
| 212 | |
|---|
| 213 | $chan_map[$colnum] = $this_chan; |
|---|
| 214 | #printf "got channel '$this_chan' in column $colnum\n"; |
|---|
| 215 | } |
|---|
| 216 | next; |
|---|
| 217 | } |
|---|
| 218 | |
|---|
| 219 | # These must be the program details |
|---|
| 220 | my $colnum = 0; |
|---|
| 221 | foreach my $session ($row->look_down('_tag' => 'td')) { |
|---|
| 222 | $colnum++; |
|---|
| 223 | if ($colnum == 1) { |
|---|
| 224 | # This session has the day name in it |
|---|
| 225 | my $newdayname = $session->as_text(); |
|---|
| 226 | $newdayname =~ s/Night//; |
|---|
| 227 | $newdayname =~ s/Day//; |
|---|
| 228 | $newdayname =~ s/Sat/Saturday/; |
|---|
| 229 | $newdayname = "" if ($dayname =~ m/Subject to avail/i); |
|---|
| 230 | |
|---|
| 231 | $dayname = $newdayname if ($newdayname =~ m/day$/); |
|---|
| 232 | next; |
|---|
| 233 | } |
|---|
| 234 | |
|---|
| 235 | my $this_chan = $chan_map[$colnum]; |
|---|
| 236 | |
|---|
| 237 | my $sessionshows = $session->as_HTML(); |
|---|
| 238 | $sessionshows =~ s/<p>/<br \/>/g; # Convert new paragraphs to newlines because they're not always consistent |
|---|
| 239 | $sessionshows =~ s/<\/p>//g; |
|---|
| 240 | $sessionshows =~ s/<u>//g; |
|---|
| 241 | $sessionshows =~ s/<\/u>//g; |
|---|
| 242 | $sessionshows =~ s/<\/td>//g; |
|---|
| 243 | $sessionshows =~ s/<\/font>//g; |
|---|
| 244 | $sessionshows =~ s/<font.*?>//g; |
|---|
| 245 | $sessionshows =~ s/<td.*?>//g; |
|---|
| 246 | #print "\nFound " . $session->as_HTML() . "\n\n"; |
|---|
| 247 | my @shows = split(/<br[\s\/]*>/,$sessionshows); |
|---|
| 248 | foreach my $show (@shows) { |
|---|
| 249 | # Bodgy stuff to fix html markup |
|---|
| 250 | $show =~ s/\&/\&/g; $show =~ s/\'/\'/g; $show =~ s/\"/\"/g; |
|---|
| 251 | $show =~ s/\¸/\,/g; |
|---|
| 252 | $show =~ s/\'/\'/g; |
|---|
| 253 | $show =~ s/\ /\ /g; |
|---|
| 254 | |
|---|
| 255 | # Remove trailing asterisk - just means could be widescreen if content is in widescreen - duh! |
|---|
| 256 | $show =~ s/\*$//; |
|---|
| 257 | |
|---|
| 258 | # High Definition and locality |
|---|
| 259 | my $hd = 0; |
|---|
| 260 | if ($show =~ m/\(HD\)\s*? \(([SMAPB])\)/) { |
|---|
| 261 | $hd = 1 if ($1 eq $opt->{locality}); |
|---|
| 262 | $show =~ s/\(HD\)\s*? \([SMAPB]\)//; |
|---|
| 263 | } |
|---|
| 264 | |
|---|
| 265 | |
|---|
| 266 | # High Definition |
|---|
| 267 | if ($show =~ m/\(HD\)/ || $show =~ m/\(JHD\)/ ) { |
|---|
| 268 | $hd = 1; |
|---|
| 269 | $show =~ s/\(HD\)//; |
|---|
| 270 | $show =~ s/\(JHD\)//; # JHD was probably a typo but we've included it here because they'll probably have fat fingers again |
|---|
| 271 | } |
|---|
| 272 | |
|---|
| 273 | # Dolby Digital 5.1 |
|---|
| 274 | my $dd51 = 0; |
|---|
| 275 | if ($show =~ m/DD 5.1/ || $show =~ m/DD5.1/) { |
|---|
| 276 | $dd51 = 1; |
|---|
| 277 | $show =~ s/\[\+ DD 5.1 sound\]//; |
|---|
| 278 | $show =~ s/\[\+ DD 5.1 Audio\]//; |
|---|
| 279 | $show =~ s/\[\+ DD 5.1 audio\]//; |
|---|
| 280 | $show =~ s/\[\+DD 5.1 audio\]//; |
|---|
| 281 | $show =~ s/\[\+DD 5.1 Audio\]//; |
|---|
| 282 | $show =~ s/\[\+DD 5.1 sound\]//; |
|---|
| 283 | $show =~ s/\[\+DD 5.1 sound\]//; |
|---|
| 284 | $show =~ s/\[\+DD5.1 sound\]//; |
|---|
| 285 | $show =~ s/\[\+DD5.1\]//; |
|---|
| 286 | $show =~ s/\[\+ DD 5.1\]//; |
|---|
| 287 | $show =~ s/\[\DD5.1\]//; |
|---|
| 288 | $show =~ s/\[\DD 5.1\]//; |
|---|
| 289 | } |
|---|
| 290 | $show =~ s/\s+$//; |
|---|
| 291 | $show =~ s/^\s+//; |
|---|
| 292 | |
|---|
| 293 | if (defined($translation->{$show})) { |
|---|
| 294 | $show = $translation->{$show}; |
|---|
| 295 | $stats{manual_translations}++; |
|---|
| 296 | } |
|---|
| 297 | |
|---|
| 298 | next if (length($show) == 0); |
|---|
| 299 | |
|---|
| 300 | # Widescreen handling |
|---|
| 301 | # Now we're here, this show must be in widescreen - we don't (yet) do anything with widescreen in mythtv |
|---|
| 302 | |
|---|
| 303 | # DD5.1 |
|---|
| 304 | if ($dd51 == 1) { |
|---|
| 305 | # This show is is in DD5.1 - we don't (yet) do anything wiht DD5.1 in mythtv |
|---|
| 306 | ; |
|---|
| 307 | } |
|---|
| 308 | |
|---|
| 309 | # High Definition Handling |
|---|
| 310 | if ($hd == 1) { |
|---|
| 311 | #print "HD Channel $this_chan $dayname \"$show\"\n" if $opt->{debug}; |
|---|
| 312 | my $xmlid = $channels->{$this_chan}; |
|---|
| 313 | $hdprog->{$xmlid}->{$show}->{chan} = $this_chan; |
|---|
| 314 | $hdprog->{$xmlid}->{$show}->{days}->{$dayname} = 1; |
|---|
| 315 | $stats{parsed_hd_progs}++; |
|---|
| 316 | } else { |
|---|
| 317 | #print "SD Widescreen Channel $channels[$channelnumber]: $dayname $show \n" if $opt->{debug}; |
|---|
| 318 | $stats{parsed_sd_widescreen_progs}++; |
|---|
| 319 | } |
|---|
| 320 | } |
|---|
| 321 | } |
|---|
| 322 | } |
|---|
| 323 | } |
|---|
| 324 | } |
|---|
| 325 | |
|---|
| 326 | ###################################################################################################### |
|---|
| 327 | # logic to fetch a page via http |
|---|
| 328 | # retries up to 3 times to get a page with 5 second pauses inbetween |
|---|
| 329 | |
|---|
| 330 | sub get_url |
|---|
| 331 | { |
|---|
| 332 | my($url,$status,$dontretry) = (@_); |
|---|
| 333 | my $response; |
|---|
| 334 | my $attempts = 0; |
|---|
| 335 | my ($raw, $page, $base); |
|---|
| 336 | |
|---|
| 337 | my $request = HTTP::Request->new(GET => $url); |
|---|
| 338 | $request->header('Accept-Encoding' => 'gzip'); |
|---|
| 339 | |
|---|
| 340 | print "$status $url\n"; |
|---|
| 341 | |
|---|
| 342 | my $max_retries = 1 if ($dontretry); |
|---|
| 343 | |
|---|
| 344 | for (1..3) { |
|---|
| 345 | $response = $ua->request($request); |
|---|
| 346 | last if ($response->is_success || $dontretry); |
|---|
| 347 | |
|---|
| 348 | $stats{http_failed_requests}++; |
|---|
| 349 | $attempts++; |
|---|
| 350 | &sleepy("attempt $attempts failed (url $url), sleeping for 10 seconds",10); |
|---|
| 351 | } |
|---|
| 352 | if (!($response->is_success)) { |
|---|
| 353 | if ($dontretry == 0) { |
|---|
| 354 | printf "ERROR: could not open url %s in %d attempts\n",$url,$attempts; |
|---|
| 355 | } |
|---|
| 356 | return undef; |
|---|
| 357 | } |
|---|
| 358 | |
|---|
| 359 | $stats{bytes_fetched} += do {use bytes; length($response->content)}; |
|---|
| 360 | $stats{http_successful_requests}++; |
|---|
| 361 | |
|---|
| 362 | if ($response->header('Content-Encoding') && |
|---|
| 363 | $response->header('Content-Encoding') eq 'gzip') { |
|---|
| 364 | $stats{compressed_pages} += do {use bytes; length($response->content)}; |
|---|
| 365 | $response->content(Compress::Zlib::memGunzip($response->content)); |
|---|
| 366 | } |
|---|
| 367 | |
|---|
| 368 | if ($response->header('Content-type') eq 'xapplication/ywe-octet-stream') { |
|---|
| 369 | $stats{transformed_pages}++; |
|---|
| 370 | $base = &transform_output(length($response->content), $response->content); |
|---|
| 371 | } else { |
|---|
| 372 | $base = $response->content; |
|---|
| 373 | } |
|---|
| 374 | return $base; |
|---|
| 375 | } |
|---|
| 376 | |
|---|
| 377 | ###################################################################################################### |
|---|
| 378 | |
|---|
| 379 | sub print_stats |
|---|
| 380 | { |
|---|
| 381 | printf "%s v%s completed", $progname, $version; |
|---|
| 382 | foreach my $key (sort keys %stats) { |
|---|
| 383 | printf ", %d %s",$stats{$key},$key; |
|---|
| 384 | } |
|---|
| 385 | printf "\n"; |
|---|
| 386 | } |
|---|
| 387 | |
|---|
| 388 | ###################################################################################################### |
|---|
| 389 | |
|---|
| 390 | sub channel_cb( $ ) |
|---|
| 391 | { |
|---|
| 392 | my $c = shift; |
|---|
| 393 | # printf "got channel ".Dumper($c); |
|---|
| 394 | $writer->write_channel($c); |
|---|
| 395 | } |
|---|
| 396 | |
|---|
| 397 | ###################################################################################################### |
|---|
| 398 | |
|---|
| 399 | sub programme_cb( $ ) |
|---|
| 400 | { |
|---|
| 401 | my $prog=shift; |
|---|
| 402 | my $this_chan = $prog->{channel}; |
|---|
| 403 | my $this_title; |
|---|
| 404 | my $hd_chan_base; |
|---|
| 405 | |
|---|
| 406 | $this_title = $prog->{title}->[0]->[0] |
|---|
| 407 | if (($prog->{title}) && ($prog->{title}->[0]) && ($prog->{title}->[0]->[0])); |
|---|
| 408 | # print "got programme ".Dumper($prog); |
|---|
| 409 | |
|---|
| 410 | goto NOMATCH if (!defined $this_chan); |
|---|
| 411 | goto NOMATCH if (!defined $this_title); |
|---|
| 412 | goto NOMATCH if (!defined $hdprog->{$this_chan}); |
|---|
| 413 | |
|---|
| 414 | # see if title matches everything we have seen on this channel |
|---|
| 415 | foreach my $t (keys %{($hdprog->{$this_chan})}) { |
|---|
| 416 | if (canonicalizeTitles_match($t,$this_title)) { |
|---|
| 417 | # match! |
|---|
| 418 | printf "Matched '$t' to '$this_title'\n"; |
|---|
| 419 | $stats{matched_prog}++; |
|---|
| 420 | $hd_chan_base = $hdprog->{$this_chan}->{$t}->{chan}."HD"; |
|---|
| 421 | goto MATCH; |
|---|
| 422 | } |
|---|
| 423 | } |
|---|
| 424 | |
|---|
| 425 | # see if title matches something in our override table |
|---|
| 426 | foreach my $ch (keys %{($hdoverride)}) { |
|---|
| 427 | next if ($channels->{$ch} ne $this_chan); |
|---|
| 428 | foreach my $t (@{($hdoverride->{$ch})}) { |
|---|
| 429 | if (canonicalizeTitles_match($t,$this_title)) { |
|---|
| 430 | # match! |
|---|
| 431 | printf "Matched '$t' to '$this_title' (hd override)\n"; |
|---|
| 432 | $stats{override_matched_prog}++; |
|---|
| 433 | $hd_chan_base = $ch."HD"; |
|---|
| 434 | goto MATCH; |
|---|
| 435 | } |
|---|
| 436 | } |
|---|
| 437 | } |
|---|
| 438 | |
|---|
| 439 | NOMATCH: |
|---|
| 440 | $writer->write_programme($prog); |
|---|
| 441 | return; |
|---|
| 442 | |
|---|
| 443 | MATCH: |
|---|
| 444 | $prog->{video}->{aspect} = "16:9"; # widescreen |
|---|
| 445 | $prog->{video}->{quality} = "HDTV"; # HD |
|---|
| 446 | $writer->write_programme($prog); |
|---|
| 447 | $stats{rewrote_prog_as_hdtv_inplace}++; |
|---|
| 448 | |
|---|
| 449 | # do we have a HD version of this known in opt_channels table? |
|---|
| 450 | if (defined $opt_channels->{$hd_chan_base}) { |
|---|
| 451 | # populate this programme in HD channel also |
|---|
| 452 | $prog->{channel} = $opt_channels->{$hd_chan_base}; |
|---|
| 453 | $writer->write_programme($prog); |
|---|
| 454 | $stats{wrote_prog_into_hdtv_channel}++ |
|---|
| 455 | } |
|---|
| 456 | return; |
|---|
| 457 | } |
|---|
| 458 | |
|---|
| 459 | ###################################################################################################### |
|---|
| 460 | |
|---|
| 461 | sub canonicalizeTitle |
|---|
| 462 | { |
|---|
| 463 | my $title=shift; |
|---|
| 464 | $title =~ s/^\s+//; |
|---|
| 465 | $title =~ s/\s+$//; |
|---|
| 466 | $title =~ s/&(#(\d+)|(.*?));/ $2 ? chr($2) : $amp{$3}||' ' /eg; |
|---|
| 467 | $title =~ s/ *\& */ and /g; |
|---|
| 468 | $title =~ s/[^ a-zA-Z0-9]//g; |
|---|
| 469 | $title =~ s/\s+/ /; |
|---|
| 470 | return(lc($title)); |
|---|
| 471 | } |
|---|
| 472 | |
|---|
| 473 | my %alternatives = ( one => "1", |
|---|
| 474 | two => "2", |
|---|
| 475 | to => "2", |
|---|
| 476 | too => "2", |
|---|
| 477 | three => "3", |
|---|
| 478 | four => "4", |
|---|
| 479 | for => "4", |
|---|
| 480 | five => "5", |
|---|
| 481 | six => "6", |
|---|
| 482 | seven => "7", |
|---|
| 483 | eight => "8", |
|---|
| 484 | nine => "9" |
|---|
| 485 | ); |
|---|
| 486 | |
|---|
| 487 | sub forgivingMatch |
|---|
| 488 | { |
|---|
| 489 | my $word1 = shift; |
|---|
| 490 | my $word2 = shift; |
|---|
| 491 | |
|---|
| 492 | # exact match |
|---|
| 493 | return 1 if $word1 eq $word2; |
|---|
| 494 | # they match according to our alternatives lookup table |
|---|
| 495 | return 1 if $alternatives{$word1} && $alternatives{$word1} eq $word2 || |
|---|
| 496 | $alternatives{$word2} && $alternatives{$word2} eq $word1; |
|---|
| 497 | # irreconcilable differences |
|---|
| 498 | return 0 if abs(length($word1) - length($word2)) > 1 || |
|---|
| 499 | length($word1) < 3; |
|---|
| 500 | |
|---|
| 501 | my @list1 = split(//,$word1); |
|---|
| 502 | my @list2 = split(//,$word2); |
|---|
| 503 | my $i = 0; |
|---|
| 504 | my $j = 0; |
|---|
| 505 | |
|---|
| 506 | # find first difference |
|---|
| 507 | while ($i < @list1 && $j < @list2 && $list1[$i] eq $list2[$j]) { |
|---|
| 508 | ++$i; ++$j; |
|---|
| 509 | } |
|---|
| 510 | if ($i+1 < @list1 && $j+1 < @list2) { |
|---|
| 511 | # at least 2 chars to go in both words |
|---|
| 512 | if ($list1[$i+1] eq $list2[$j] && $list1[$i] eq $list2[$j+1]) { |
|---|
| 513 | # characters transposed |
|---|
| 514 | $i += 2; |
|---|
| 515 | $j += 2; |
|---|
| 516 | } elsif ($list1[$i+1] eq $list2[$j]) { |
|---|
| 517 | # extra character inserted into @list1 |
|---|
| 518 | $i += 2; |
|---|
| 519 | ++$j; |
|---|
| 520 | } elsif ($list1[$i] eq $list2[$j+1]) { |
|---|
| 521 | # extra character inserted into @list2 |
|---|
| 522 | ++$i; |
|---|
| 523 | $j += 2; |
|---|
| 524 | } else { |
|---|
| 525 | # single character difference |
|---|
| 526 | ++$i; |
|---|
| 527 | ++$j; |
|---|
| 528 | } |
|---|
| 529 | # we forgave one difference; now do rest of strings match exactly? |
|---|
| 530 | while ($i < @list1 && $j < @list2 && $list1[$i] eq $list2[$j]) { |
|---|
| 531 | ++$i; ++$j; |
|---|
| 532 | } |
|---|
| 533 | return($i == @list1 && $j == @list2); |
|---|
| 534 | } elsif ($i == @list1 || $j == @list2) { |
|---|
| 535 | # only difference is one word has one extra letter, or last char |
|---|
| 536 | # of each word differ. That's still only one one-char difference |
|---|
| 537 | return(1); |
|---|
| 538 | } |
|---|
| 539 | } |
|---|
| 540 | |
|---|
| 541 | sub canonicalizeTitles_match |
|---|
| 542 | { |
|---|
| 543 | my $word1=canonicalizeTitle(shift); |
|---|
| 544 | my $word2 =canonicalizeTitle(shift); |
|---|
| 545 | my @longer; |
|---|
| 546 | my @shorter; |
|---|
| 547 | |
|---|
| 548 | if (length($word1) > length($word2)) { |
|---|
| 549 | @longer = split(/\s+/, $word1); |
|---|
| 550 | @shorter = split(/\s+/, $word2); |
|---|
| 551 | } else { |
|---|
| 552 | @shorter = split(/\s+/, $word1); |
|---|
| 553 | @longer = split(/\s+/, $word2); |
|---|
| 554 | } |
|---|
| 555 | |
|---|
| 556 | WORD: for my $word (@shorter) { |
|---|
| 557 | for(my $i=0; $i < @longer; ++$i) { |
|---|
| 558 | if (forgivingMatch($longer[$i], $word)) { |
|---|
| 559 | splice(@longer,$i,1); |
|---|
| 560 | next WORD; |
|---|
| 561 | } elsif ($i+1 < @longer && |
|---|
| 562 | $word eq "$longer[$i]$longer[$i+1]") { |
|---|
| 563 | splice(@longer,$i,2); |
|---|
| 564 | next WORD; |
|---|
| 565 | } |
|---|
| 566 | } |
|---|
| 567 | return(0); |
|---|
| 568 | } |
|---|
| 569 | return(1); |
|---|
| 570 | } |
|---|
| 571 | |
|---|
| 572 | ############################################################################## |
|---|
