my $calls_dir3 = "Bing/1Parsed/Html2/";
opendir( my $search_dir2, $calls_dir3 ) or die "$!\n";
my @files = grep /\.txt$/i, readdir $search_dir2;
closedir $search_dir2;
print "Got ", scalar @files, " files\n";
# proxies
open my $fh2, '<', 'proxies.txt' or die $!;
chomp( my @proxies = <$fh2> );
close $fh2;
foreach my $file (@files) {
my %seen = ();
my $current_file = $calls_dir3 . $file;
my $proxy = shift @proxies;
$proxy;
print "Current proxy:$proxy\n";
open my $FILE, '<', $current_file or die "$file: $!\n";
my $x=1;
make_path('Bing/1Parsed/Html3/');
while ( my $row = <$FILE> ) {
open my $fh1, ">", "Bing/1Parsed/Html3/$file.$x.html"
or die("Could not open file. $!");
#my $rnumber2 = rand(1999);
$x = $x + 1;
chomp $row;
print "$row\n";
my $xml1 = $row;
$fh1->print ("");
# create useragent
my $ua = LWP::UserAgent->new;
$ua->agent('Mozilla/5.0');
$ua->timeout('180');
# Use this UA/Proxy to fetch something....
$ua->proxy( ['http'], 'http://' . $proxy );
my $xml2 = get $xml1;
$xml2;
#add pause
#my ( $x, $y ) = ( 1, 2 );
#my $result = int( rand( $y - $x + 1 ) ) + $x;
#print "pausing ";
#print $result;
#print "seconds";
#sleep ($result);
$fh1->print("\n");
print $xml2;
$fh1->print($xml2);
close $fh1;
$xml2 = 1;
}
}