#!perl
use strict;
use LWP::UserAgent;
use File::Path 'make_path';

my $link1='';
my $link2='';
my $link3='';

# make directories
my $dir = 'Bing/1Parsed/Html/';
unless (-d $dir){
  make_path($dir, { verbose=>1, mode=>0755 });
}

# keywords
open my $fh, '<', 'keywords.txt' or die $!;
chomp(my @keywords = <$fh>);
close $fh;

# cities
open my $fh, '<', 'cities.txt' or die $!;
chomp(my @cities = <$fh>);
close $fh;

# proxies
open my $fh, '<', 'proxies.txt' or die $!;
chomp(my @proxies = <$fh>);
close $fh;
  
# create useragent
my $ua = LWP::UserAgent->new; 
$ua->agent('Mozilla/8.0');

# get pages for each city, keyword, proxy 
for my $city (@cities) {
  for my $keyword (@keywords){
    print "City :    [$city]\nKeyword: [$keyword]\n";

    for my $i (0..$#proxies){
      print "Current proxy:$proxies[$i]\n";

      # Use this UA/Proxy to fetch something....
      $ua->proxy(['http'], 'http://'.$proxies[$i]);     
      my $url = join '',$link1 . $city ."+". $keyword,
                        $link2 . $city ."+". $keyword,
                        $link3;
      my $response = $ua->get($url);
      print "getting $url\n";      
      if ($response->is_success) {
        my $filename = "Bing/${city}_${keyword}_${i}.html";
        print "Creating $filename\n";
        open my $fh, ">", $filename 
             or die("Could not open $filename. $!");
        print $fh $response->decoded_content;  # or whatever
        close $fh;
      } else {
        die $response->status_line;
      }
      print "\n"; 
    }

  }
}