#!/usr/bin/perl
use lib ".";
use Spider::LWP; #Overloaded version of LWP
use HTTP::Request;
use Googly::Search;
use Getopt::Long;
use File::Slurp;
use Data::Dump qw(dump);

sub main{

my @list = $ARGV[0]; #starting dir of lists of participating URLS.

foreach(@list){
                if($_ !~ /\w/){next;}
                chomp;
                $urls = GetListOfSta
rtingURLS($_);
                my $path_addition = $_;
                $path_addition =~ s/[\.\s+\\\/\:\*\"\?\<\>\|]+/_/g;
                $path_addition =~ s/[\_\s*]$//;
                foreach my $url(@$urls){
                        my $req = HTTP::Request->new('GET', $url);
                        $ua->{current}->{$req->uri->as_string} = Spider::URL->new($req->uri->as_string,$path_addition,0);
                        $req->header('Accept-Encoding'=>'gzip,x-gzip,x-bzip2,deflate,compress,base64,quoted-printable');
                        $ua->register ($req);
                }
        }
        $SIG{INT} = sub {
#print dump $ua;
print_queue($ua->{queue},"QUEUE");
print_queue($ua->{current},"CURRENT");
print_queue($ua->{failed},"FAILED");
print_queue($ua->{finished},"FINISHED");
exit;
        };
       $ua->wait(300); # block until we are all finished or until everything has stopped for 5 minutes
}