Changeset 69

Show
Ignore:
Timestamp:
10/13/06 21:26:58 (7 years ago)
Author:
lincoln
Message:

update jrobbo to 0.03: bugfix: fix ABC2 xmltvid, feature: say who we are (fetch direct and not via webwarper), put ourselves in the user-agent field

Files:
2 modified

Legend:

Unmodified
Added
Removed

  • grabbers/jrobbo

    r61 r69  
    88 
    99my $progname = "jrobbo"; 
    10 my $version = "0.02"; 
     10my $version = "0.03"; 
    1111 
    1212use LWP::UserAgent; 
    … …  
    5454        'output=s'      => \$opt->{outputfile}, 
    5555        'downloadfile=s' => \$opt->{downloadfile}, 
    56         'nowarper'      => \$opt->{nowarper}, 
     56        'warper'        => \$opt->{warper}, 
     57        'anonymous'     => \$opt->{anonymous}, 
    5758        'help'          => \$opt->{help}, 
    5859        'version'       => \$opt->{version}, 
    … …  
    7374        printf "   --output=<file>        file to send output to (default $opt->{outputfile})\n"; 
    7475        printf "   --downloadfile=<file>  file to send output to (default $opt->{downloadfile})\n"; 
    75         printf "   --nowarper              don't fetch via webwarper\n"; 
     76        printf "   --warper               fetch via webwarper (default: don't)\n"; 
     77        printf "   --anonymous            fetch anonymously (default: don't)\n"; 
    7678        printf "\n"; 
    7779        printf "  regions are as follows:\n"; 
    … …  
    104106} 
    105107 
    106 printf "Fetching %s %s...\n",$url,($opt->{nowarper} ? "direct" : "via webwarper"); 
    107 $url =~ s#^http://#http://webwarper.net/ww/# unless $opt->{nowarper}; 
     108printf "Fetching %s%s%s...\n",$url,($opt->{anonymous} ? "anonymously " : ""),($opt->{warper} ? "via webwarper" : ""); 
     109$url =~ s#^http://#http://webwarper.net/ww/# unless $opt->{warper}; 
    108110 
    109111my $agent = ( 
    … …  
    114116        'Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/412 (KHTML, like Gecko) Safari/412' 
    115117        )[int(rand(5))]; 
     118$agent = "Shepherd/$progname $version" if (!$opt->{anonymous}); 
    116119my $ua = LWP::UserAgent->new('timeout' => 30, 'keep_alive' => 1, 'agent' => $agent); 
    117120$ua->env_proxy; 
    … …  
    119122 
    120123my $request = HTTP::Request->new(GET => $url); 
    121 my $randomaddr = sprintf "203.%d.%d.%d",rand(255),rand(255),(rand(254)+1); 
    122 $request->header('Via' => '1.0 proxy:81 (Squid/2.3.STABLE3)'); 
    123 $request->header('X-Forwarded-For' => $randomaddr); 
     124 
     125if ($opt->{anonymous}) { 
     126        my $randomaddr = sprintf "203.%d.%d.%d",rand(255),rand(255),(rand(254)+1); 
     127        $request->header('Via' => '1.0 proxy:81 (Squid/2.3.STABLE3)'); 
     128        $request->header('X-Forwarded-For' => $randomaddr); 
     129} 
     130 
    124131my $response; 
    125132 
    … …  
    149156                my $newchan = $chan; 
    150157                if ($chan =~ /^ABC /)                   { $newchan = $channels->{"ABC"}; } 
     158                elsif ($chan =~ /^ABC2/)                { $newchan = $channels->{"ABC2"}; } 
    151159                elsif ($chan =~ /^Channel Seven/)       { $newchan = $channels->{"Seven"}; } 
    152160                elsif ($chan =~ /^Prime/)               { $newchan = $channels->{"Seven"}; } 

  • status

    r68 r69  
    55abc2_website:1.55-r2:grabber 
    66yahoo7widget:1.53-r2:grabber 
    7 jrobbo:0.02-r2:grabber 
     7jrobbo:0.03-r2:grabber 
    88d1:0.6.2.3-r2:grabber 
    99imdb_augment_data:0.01:postprocessor