[16:57:57.93] P:\test>418095 >nul
Queued urls: 2270


Time:97
Done

##</code><code>##

#!/usr/bin/perl
use strict;
use threads;
use threads::shared;
use LWP::UserAgent;
use HTTP::Request::Common;
use Thread::Queue;

$| = 1;

my $thread_num : shared = 0;
my $max_thread = 25;
my $exit = 0;
my $dump = 0;
my $start_time = 0;
my %tid : shared = ();
my $task_q = Thread::Queue->new();
my $result_q = Thread::Queue->new();


my @urls = <DATA>; chomp @urls;
$task_q->enqueue( @urls );
undef @urls;
warn "Queued urls: ", $task_q->pending, "\n";

$start_time = time();

threads->new(\&thread_do) for (1..$max_thread);

while ( $task_q->pending ) {
	select undef, undef, undef, 0.1;
	print $result_q->dequeue(), "\n" while $result_q->pending();

	if ($dump++ > 100000) {
#		print "Dump\n";
		dump_tid(\%tid);
		$dump = 0;
	}
}

sleep 3; ## Give the task threads time to finish up
warn "\n\nTime:" . (time() - $start_time) . "\n";
warn "Done\n";


sub thread_do
{
	threads->self->detach();
	my $tid = threads->self->tid();
	my $ua = LWP::UserAgent->new( timeout => 3 );
	while ( $task_q->pending ) {       
		my $url = $task_q->dequeue;
		my $res = $ua->request( HEAD $url );
		$result_q->enqueue( "$tid; $url ::= " . $res->code() . ";" );

		lock %tid;
		$tid{ $tid }++;
	}
}


sub dump_tid
{
#    my $tid = shift;
#    open (DUMP, "> dump.txt");
#    print DUMP "$_ = $tid->{$_}\n" foreach keys %$tid;
#    close DUMP;
}