#!/usr/bin/perl

 require LWP::UserAgent;
 use HTML::FormatText;
 use Time::HiRes;
 require HTML::TreeBuilder;

print "Opening fetch file...\n";

#Open the DMOZ file
open(tmp, "Top/Arts/Movies/domains");
  @domains = <tmp>;
close(tmp);

#set variables used to output later
$count=0;
$interval=1;

foreach(@domains) {
	$count++;
	$domain = "http://" . $_;

	$start = Time::HiRes::time();

        my $ua = LWP::UserAgent->new;
        $ua->timeout(10);
       	$ua->env_proxy;
 
 	my $response = $ua->get($domain);
 
 	if ($response->is_success) {
     		$content =  $response->content;  # or whatever
 	}
 	else {
     		$response->status_line;
 	}

	$tree = HTML::TreeBuilder->new->parse_file("file.html");
	$formatter = HTML::FormatText->new(leftmargin => 0, rightmargin => 50);

        $end = Time::HiRes::time();
	$time = $end-$start;
	$total += $time;
	if($total >=(10*$interval) ) {
		print "$count sites in $total seconds\n";
		$interval++;
	}
}