Changeset 754

Timestamp:

06/09/07 02:35:05 (6 years ago)

Author:

paul

Message:

allow tvdb_augment_data parser to attempt fixups and handle failure

Files:

• postprocessors/tvdb_augment_data (modified) (6 diffs)
• status (modified) (1 diff)

postprocessors/tvdb_augment_data

@@ -20 +20 @@
 
 my $progname = "tvdb_augment_data";
-my $version = "0.03";
+my $version = "0.04";
 my $mirrorlist_url = 'http://thetvdb.com/interfaces/GetMirrors.php';
 
@@ -493 +493 @@
 
 ##############################################################################
+# parser xml
+
+sub parse
+{
+        my $data = shift;
+
+        my $xml_tree;
+        eval { $xml_tree = $parser->parse($data); };
+        if (!$xml_tree) {
+                Shepherd::Common::log("  xml parse failed, attempting to fixup any missing ;");
+                # fix bad entities without ending ; by adding ; like "anc&#233, Trent"
+                $data =~ s/&(amp|quot|gt|lt|#\d+|#x[0-9a-f]+)(?!;)/&$1;/gsi;
+
+                eval { $xml_tree = $parser->parse($data); };
+                if (!$xml_tree) {
+                        Shepherd::Common::log("  xml parse of fixup failed, attempting to elimination any &");
+                        # ignore bad entities without ending ; like "anc&#233, Trent"
+                        $data =~ s/&/and/g;     
+
+                        eval { $xml_tree = $parser->parse($data); };
+                }
+        }
+
+        return $xml_tree;
+}
+
+##############################################################################
 # process all xmltv files
 
@@ -598 +625 @@
         my $data = get_url(url => $mirrorlist_url);
         die "could not gather list of mirrors from $mirrorlist_url\n" if (!$data);
-
-        my $xml_tree = $parser->parse($data);
+        my $xml_tree = &parse($data);
+        die "could not parse list of mirrors from $mirrorlist_url\n" if (!$xml_tree);
+
         $d->{tvdb}->{num_mirrors} = $xml_tree->getElementsByTagName("Item")->getLength;
         for (my $i = 0; $i < $d->{tvdb}->{num_mirrors}; $i++) {
@@ -705 +733 @@
                 return;
         }
-
-        my $xml_tree = $parser->parse($data);
+        my $xml_tree = &parse($data);
+        if (!$xml_tree) {
+                Shepherd::Common::log("   series parse of ".$d->{tvdb}->{mirror}.$url." failed");
+                $stats{prog_lookup_failed}++;
+                return;
+        }
+
         my $num_items = $xml_tree->getElementsByTagName("Item")->getLength;
 
@@ -789 +822 @@
                         Shepherd::Common::log("   series detail lookup request of ".$url." failed");
                         $stats{series_update_lookup_failed}++;
-                        return;
-                }
-
-                my $xml_tree = $parser->parse($data);
+                        next;
+                }
+                my $xml_tree = &parse($data);
+                if (!$xml_tree) {
+                        Shepherd::Common::log("   series detail lookup parse of ".$url." failed");
+                        $stats{series_update_lookup_failed}++;
+                        next;
+                }
+
                 my $num_items = $xml_tree->getElementsByTagName("Item")->getLength;
 
@@ -850 +888 @@
                 my $data = get_url(url => $url);
                 if (!$data) {
-                        Shepherd::Common::log("   epside detail lookup request of ".$url." failed");
+                        Shepherd::Common::log("   episode detail lookup request of ".$url." failed");
                         $stats{episode_update_lookup_failed}++;
-                        return;
-                }
-
-                my $xml_tree = $parser->parse($data);
+                        next;
+                }
+                my $xml_tree = &parse($data);
+                if (!$xml_tree) {
+                        Shepherd::Common::log("   episode detail lookup parse of ".$url." failed");
+                        $stats{episode_update_lookup_failed}++;
+                        next;
+                }
+
                 my $num_items = $xml_tree->getElementsByTagName("Item")->getLength;
 

status

@@ -16 +16 @@
 reconciler      reconciler_mk2      0.27
 postprocessor   imdb_augment_data   0.19
-postprocessor   tvdb_augment_data   0.03
+postprocessor   tvdb_augment_data   0.04
 postprocessor   flag_aus_hdtv       0.17
 postprocessor   augment_timezone    0.14