| 395 | | Shepherd::Common::log(" [chose mirror ".$d->{tvdb}->{mirror}."]"); |
| 396 | | |
| 397 | | } |
| 398 | | |
| 399 | | ############################################################################## |
| 400 | | |
| 401 | | sub lookup_title |
| 402 | | { |
| 403 | | my ($title, $subtitle) = @_; |
| 404 | | |
| 405 | | &lookup_mirrors if (!defined $d->{tvdb}->{mirrors}); |
| | 421 | Shepherd::Common::log(" chose mirror ".$d->{tvdb}->{mirror}." for data"); |
| | 422 | |
| | 423 | } |
| | 424 | |
| | 425 | ############################################################################## |
| | 426 | # find a 'seriesid' associated with this title |
| | 427 | # (actually just populates $data_cache->{title}->{$lc_title}) |
| | 428 | |
| | 429 | sub lookup_title_web |
| | 430 | { |
| | 431 | my ($title, $prog_count) = @_; |
| | 432 | my $lc_title = lc($title); |
| | 433 | |
| | 434 | my $letter = substr($lc_title,0,1); |
| | 435 | $letter = "OTHER" if ($letter !~ /[a-z]/); |
| | 436 | |
| | 437 | my $found_title = 0; |
| | 438 | |
| | 439 | # only lookup if we have passed our caching threshold for this letter... |
| | 440 | if ((!defined $data_cache->{title}->{"WEB_LOOKUP_".$letter}->{expires}) || |
| | 441 | ($data_cache->{title}->{"WEB_LOOKUP_".$letter}->{expires} < time)) { |
| | 442 | Shepherd::Common::log(" fetching series tables for '".$letter."' ..."); |
| | 443 | |
| | 444 | my $url = "http://thetvdb.com/?tab=listseries&letter=".$letter; |
| | 445 | my $data = get_url(url => $url); |
| | 446 | if (!$data) { |
| | 447 | Shepherd::Common::log(" ".$url." didn't return any valid data! skipping..."); |
| | 448 | |
| | 449 | # try again in 7 days |
| | 450 | $data_cache->{title}->{"WEB_LOOKUP_".$letter}->{expires} = time + (7 * 86400); |
| | 451 | $stats{failed_title_web_fetch}++; |
| | 452 | return; |
| | 453 | } |
| | 454 | |
| | 455 | my $expires_in = time + ($opt->{cache_title_for} * 86400); |
| | 456 | $data_cache->{title}->{"WEB_LOOKUP_".$letter}->{expires} = $expires_in; |
| | 457 | |
| | 458 | my $tree = HTML::TreeBuilder->new_from_content($data); |
| | 459 | my $tree_table = $tree->look_down('_tag' => 'table', 'id' => 'listtable'); |
| | 460 | |
| | 461 | foreach my $tree_tr ($tree_table->look_down('_tag' => 'tr')) { |
| | 462 | my @tree_td = $tree_tr->look_down('_tag' => 'td'); |
| | 463 | |
| | 464 | if (((scalar @tree_td) == 3) && ($tree_td[2]->as_text() =~ /^(\d+)$/)) { |
| | 465 | my $series_name = lc($tree_td[0]->as_text()); |
| | 466 | $data_cache->{title}->{$series_name}->{SeriesName} = $tree_td[0]->as_text(); |
| | 467 | $data_cache->{title}->{$series_name}->{SeriesID} = $tree_td[2]->as_text(); |
| | 468 | $data_cache->{title}->{$series_name}->{expires} = $expires_in; |
| | 469 | $data_cache->{title}->{$series_name}->{notfound} = 0; |
| | 470 | $stats{inserted_title_into_cache}++; |
| | 471 | } |
| | 472 | } |
| | 473 | |
| | 474 | &write_cache; |
| | 475 | $tree->delete; |
| | 476 | } |
| | 477 | } |
| | 478 | |
| | 479 | |
| | 480 | ############################################################################## |
| | 481 | # find an 'episode' associated with a series |
| | 482 | |
| | 483 | sub lookup_episode |
| | 484 | { |
| | 485 | my ($lc_title, $subtitle) = @_; |
| | 486 | my $seriesid = $data_cache->{title}->{$lc_title}->{SeriesID}; |
| | 487 | my $lc_subtitle = lc($subtitle); |
| | 488 | return if (!defined $seriesid); |
| | 489 | |
| | 490 | $stats{used_title_cache_item}++; |
| | 491 | |
| | 492 | my $url = "/GetEpisodes.php?seriesid=".$seriesid."&IncludeSeriesInfo=1"; |
| | 493 | if ($lc_subtitle eq "") { |
| | 494 | $url .= "&season=1&episode=1"; |
| | 495 | $lc_subtitle = "NONE"; |
| | 496 | } else { |
| | 497 | $url .= "&episodename=".Shepherd::Common::urlify($lc_subtitle); |
| | 498 | } |
| | 499 | |
| | 500 | # return without doing anything if the entry already exists in the cache |
| | 501 | if (defined $data_cache->{prog}->{$lc_title}->{$lc_subtitle}) { |
| | 502 | $stats{used_prog_cache_item}++; |
| | 503 | goto CHECK_EPISODE if ($lc_subtitle ne "NONE"); |
| | 504 | return; |
| | 505 | } |
| | 506 | |
| | 507 | Shepherd::Common::log(" fetching series '".$data_cache->{title}->{$lc_title}->{SeriesName}."' ".($lc_subtitle ne "NONE" ? "(episode '$subtitle')" : ""). " ..."); |
| | 508 | |
| | 509 | &lookup_mirrors if (!defined $d->{tvdb}->{mirror}); |
| | 510 | my $data = get_url(url => $d->{tvdb}->{mirror}.$url); |
| | 511 | if (!$data) { |
| | 512 | Shepherd::Common::log(" series lookup of ".$d->{tvdb}->{mirror}.$url." failed"); |
| | 513 | $stats{prog_lookup_failed}++; |
| | 514 | return; |
| | 515 | } |
| | 516 | |
| | 517 | my $xml_tree = $parser->parse($data); |
| | 518 | my $num_items = $xml_tree->getElementsByTagName("Item")->getLength; |
| | 519 | |
| | 520 | if ($num_items == 1) { |
| | 521 | $stats{negatively_cached_prog}++; |
| | 522 | |
| | 523 | $data_cache->{prog}->{$lc_title}->{$lc_subtitle}->{notfound} = 1; |
| | 524 | $data_cache->{prog}->{$lc_title}->{$lc_subtitle}->{expires} = time + (($opt->{cache_details_for} / 2) * 86400); |
| | 525 | |
| | 526 | # on an episode-lookup failure, thetvdb.com doesn't return any SeriesData, |
| | 527 | # so schedule that for a bulk get. |
| | 528 | if ((!defined $data_cache->{prog}->{$lc_title}->{SERIES}) && |
| | 529 | (!defined $d->{series_lookup}->{$lc_title})) { |
| | 530 | $d->{series_lookup}->{$lc_title} = $seriesid; |
| | 531 | $d->{series_lookup_requests}++; |
| | 532 | } |
| | 533 | return; |
| | 534 | } |
| | 535 | |
| | 536 | my $item = $xml_tree->getElementsByTagName("Item")->item(1); |
| | 537 | |
| | 538 | # remove existing entries |
| | 539 | delete $data_cache->{prog}->{$lc_title}->{SERIES} if (defined $data_cache->{prog}->{$lc_title}->{SERIES}); |
| | 540 | delete $data_cache->{prog}->{$lc_title}->{$lc_subtitle} if (defined $data_cache->{prog}->{$lc_title}->{$lc_subtitle}); |
| | 541 | |
| | 542 | # set expiry on these |
| | 543 | $data_cache->{prog}->{$lc_title}->{SERIES}->{expires} = time + ($opt->{cache_details_for} * 86400); |
| | 544 | $data_cache->{prog}->{$lc_title}->{$lc_subtitle}->{expires} = time + ($opt->{cache_details_for} * 86400); |
| | 545 | |
| | 546 | foreach my $field ("SeriesData-Status", "SeriesData-FirstAired", "SeriesData-Network", "SeriesData-Genre", "SeriesData-Actors", "SeriesData-Overview", "id", "SeasonNumber", "EpisodeNumber", "EpisodeName") { |
| | 547 | my $fieldtag = $item->getElementsByTagName($field)->item(0)->getFirstChild; |
| | 548 | if (defined $fieldtag) { |
| | 549 | if ($field =~ /^SeriesData-(.*)$/) { |
| | 550 | $data_cache->{prog}->{$lc_title}->{SERIES}->{$1} = $fieldtag->getNodeValue; |
| | 551 | } else { |
| | 552 | $data_cache->{prog}->{$lc_title}->{$lc_subtitle}->{$field} = $fieldtag->getNodeValue; |
| | 553 | } |
| | 554 | } |
| | 555 | } |
| | 556 | $xml_tree->dispose; |
| | 557 | $stats{inserted_prog_into_cache}++; |
| | 558 | &write_cache if (($stats{inserted_prog_into_cache} % 15) == 0); |
| | 559 | |
| | 560 | CHECK_EPISODE: |
| | 561 | if ((defined $data_cache->{prog}->{$lc_title}->{$lc_subtitle}->{id}) && |
| | 562 | (!defined $data_cache->{prog}->{$lc_title}->{$lc_subtitle}->{have_episode_details})) { |
| | 563 | my $ep; |
| | 564 | $ep->{id} = $data_cache->{prog}->{$lc_title}->{$lc_subtitle}->{id}; |
| | 565 | $ep->{title} = $lc_title; |
| | 566 | $ep->{subtitle} = $lc_subtitle; |
| | 567 | |
| | 568 | push(@{($d->{episode_lookup})}, $ep); |
| | 569 | $d->{episode_lookup_requests}++; |
| | 570 | } |
| | 571 | } |
| | 572 | |
| | 573 | ############################################################################## |
| | 574 | # used to lookup SeriesData for known (good) seriesid's |
| | 575 | |
| | 576 | sub lookup_series_updates |
| | 577 | { |
| | 578 | my @id_list = values %{($d->{series_lookup})}; |
| | 579 | |
| | 580 | while ((scalar @id_list) > 0) { |
| | 581 | Shepherd::Common::log(" ".(scalar @id_list)." remaining..."); |
| | 582 | |
| | 583 | &lookup_mirrors if (!defined $d->{tvdb}->{mirror}); |
| | 584 | my $url = $d->{tvdb}->{mirror}."/SeriesUpdates.php?lasttime=0&idlist="; |
| | 585 | |
| | 586 | # grab up to 20 at a time |
| | 587 | foreach my $count (1..20) { |
| | 588 | my $id = pop(@id_list); |
| | 589 | |
| | 590 | if (defined $id) { |
| | 591 | $url .= ',' if ($count > 1); |
| | 592 | $url .= $id; |
| | 593 | } |
| | 594 | } |
| | 595 | |
| | 596 | my $data = get_url(url => $url); |
| | 597 | if (!$data) { |
| | 598 | Shepherd::Common::log(" series detail lookup request of ".$url." failed"); |
| | 599 | $stats{series_update_lookup_failed}++; |
| | 600 | return; |
| | 601 | } |
| | 602 | |
| | 603 | my $xml_tree = $parser->parse($data); |
| | 604 | my $num_items = $xml_tree->getElementsByTagName("Item")->getLength; |
| | 605 | |
| | 606 | for (my $i=1; $i < $num_items; $i++) { |
| | 607 | my $item = $xml_tree->getElementsByTagName("Item")->item($i); |
| | 608 | my $namefield = $item->getElementsByTagName("SeriesName")->item(0)->getFirstChild; |
| | 609 | |
| | 610 | if (defined $namefield) { |
| | 611 | my $name = lc($namefield->getNodeValue); |
| | 612 | |
| | 613 | foreach my $field ("Status", "FirstAired", "Network", "Genre", "Actors", "Overview") { |
| | 614 | my $fieldtag = $item->getElementsByTagName($field)->item(0)->getFirstChild; |
| | 615 | if (defined $fieldtag) { |
| | 616 | $data_cache->{prog}->{$name}->{SERIES}->{$field} = $fieldtag->getNodeValue; |
| | 617 | } |
| | 618 | } |
| | 619 | |
| | 620 | $data_cache->{prog}->{$name}->{SERIES}->{expires} = time + ($opt->{cache_details_for} * 86400); |
| | 621 | $stats{inserted_prog_into_cache}++; |
| | 622 | &write_cache if (($stats{inserted_prog_into_cache} % 15) == 0); |
| | 623 | } |
| | 624 | } |
| | 625 | |
| | 626 | $xml_tree->dispose; |
| | 627 | } |
| | 628 | |
| | 629 | &write_cache; |
| | 630 | } |
| | 631 | |
| | 632 | ############################################################################## |
| | 633 | # used to lookup episode details |
| | 634 | |
| | 635 | sub lookup_episode_updates |
| | 636 | { |
| | 637 | my $episodes; |
| | 638 | |
| | 639 | while ((scalar @{($d->{episode_lookup})}) > 0) { |
| | 640 | Shepherd::Common::log(" ".(scalar @{($d->{episode_lookup})})." remaining..."); |
| | 641 | |
| | 642 | &lookup_mirrors if (!defined $d->{tvdb}->{mirror}); |
| | 643 | my $url = $d->{tvdb}->{mirror}."/EpisodeUpdates.php?lasttime=0&idlist="; |
| | 644 | |
| | 645 | # grab up to 20 at a time |
| | 646 | foreach my $count (1..20) { |
| | 647 | my $ep = pop(@{($d->{episode_lookup})}); |
| | 648 | |
| | 649 | if (defined $ep) { |
| | 650 | my $id = $ep->{id}; |
| | 651 | $episodes->{$id}->{title} = $ep->{title}; |
| | 652 | $episodes->{$id}->{subtitle} = $ep->{subtitle}; |
| | 653 | |
| | 654 | $url .= ',' if ($count > 1); |
| | 655 | $url .= $id; |
| | 656 | } |
| | 657 | } |
| | 658 | |
| | 659 | my $data = get_url(url => $url); |
| | 660 | if (!$data) { |
| | 661 | Shepherd::Common::log(" epside detail lookup request of ".$url." failed"); |
| | 662 | $stats{episode_update_lookup_failed}++; |
| | 663 | return; |
| | 664 | } |
| | 665 | |
| | 666 | my $xml_tree = $parser->parse($data); |
| | 667 | my $num_items = $xml_tree->getElementsByTagName("Item")->getLength; |
| | 668 | |
| | 669 | for (my $i=1; $i < $num_items; $i++) { |
| | 670 | my $item = $xml_tree->getElementsByTagName("Item")->item($i); |
| | 671 | my $id_field = $item->getElementsByTagName("id")->item(0); |
| | 672 | |
| | 673 | if ((defined $id_field) && (defined $id_field->getFirstChild)) { |
| | 674 | my $id = $id_field->getFirstChild->getNodeValue; |
| | 675 | my $title = $episodes->{$id}->{title}; |
| | 676 | my $subtitle = $episodes->{$id}->{subtitle}; |
| | 677 | |
| | 678 | foreach my $field ("id", "EpisodeNumber", "EpisodeName", "FirstAired", "GuestStars", "Director", "Writer", "Overview", "ShowURL", "DVD_discid", "DVD_season", "DVD_episodenumber", "DVD_chapter", "IncorrectID") { |
| | 679 | # print "looking for $field in item $i $url\n"; |
| | 680 | my $fieldtag = $item->getElementsByTagName($field)->item(0); |
| | 681 | if ((defined $fieldtag) && (defined $fieldtag->getFirstChild)) { |
| | 682 | $data_cache->{prog}->{$title}->{$subtitle}->{$field} = $fieldtag->getFirstChild->getNodeValue; |
| | 683 | } |
| | 684 | } |
| | 685 | $data_cache->{prog}->{$title}->{$subtitle}->{have_episode_details} = 1; |
| | 686 | $data_cache->{prog}->{$title}->{$subtitle}->{expires} = time + (($opt->{cache_details_for} / 2) * 86400); |
| | 687 | |
| | 688 | $stats{inserted_episode_into_cache}++; |
| | 689 | &write_cache if (($stats{inserted_episode_into_cache} % 15) == 0); |
| | 690 | } |
| | 691 | } |
| | 692 | |
| | 693 | $xml_tree->dispose; |
| | 694 | } |
| | 695 | |
| | 696 | &write_cache; |