#! perl -slw
use strict;
use threads qw[ yield async ];
use Thread::Queue;
use LWP::Simple;

$|=1;

our $THREADS ||= 4;

my $Qwork    = new Thread::Queue;
my $Qresults = new Thread::Queue;

sub work{
	my $tid = threads->self->tid();
	sleep 1 until $Qwork->pending;
	while( $Qwork->pending or ( sleep(1) and $Qwork->pending ) ) {
		my $work = $Qwork->dequeue;
		print "$tid checking: $work";
		if( -e "c:/perlmonks/$work.xml" and not -z _ ) {
			$Qresults->enqueue( "$work returned exists not fetched" );
			next;
		}
		print "$tid fetching: $work";
		my $rc = getstore(
			"http://perlmonks.com/index.pl?node_id=$work&displaytype=xml",
			"c:/perlmonks/$work.xml"
		);

		$Qresults->enqueue( "$work returned $rc" );
	}
	return;
}

my @t = map{ threads->new( \&work ) } 1 .. $THREADS;

open IN, '<', $ARGV[ 0 ] or die "$ARGV[ 0 ] : $!";

my $cResults = 0;
while( <IN> ) {
	chomp;
	$Qwork->enqueue( $_ ) and $cResults++;
	sleep 1 while $Qwork->pending > 100;
}

$_->join for @t;

my %summary;
$Qresults->dequeue =~ m[(\S+) returned (\S+)]
	and push @{ $summary{ $2 } }, $1
	while $Qresults->pending;

for my $key ( sort keys %summary ) {
	print $key, scalar @{ $summary{ $key } };
}

printf 'Enter to see details or ^C'; <STDIN>;

for my $key ( sort keys %summary ) {
	print $key;
	my $details =  join ' ', @{ $summary{ $key } };
	$details =~ s[.{1,80} ][\n]g;
	print $details;
}