#! perl -slw
use strict;
use threads qw[ yield async ];
use Thread::Queue;
use LWP::Simple;

$|=1;

our $THREADS ||= 4;

my $Qwork    = new Thread::Queue;
my $Qresults = new Thread::Queue;

sub work{
    my $tid = threads->self->tid();
    sleep 1 until $Qwork->pending;
    while( $Qwork->pending or ( sleep(1) and $Qwork->pending ) ) {
        my $work = $Qwork->dequeue;
        print "$tid checking: $work";
        if( -e "c:/perlmonks/$work.xml" and not -z _ ) {
            $Qresults->enqueue( "$work returned exists not fetched" );
            next;
        }
        print "$tid fetching: $work";
        my $rc = getstore(
            "http://perlmonks.com/index.pl?node_id=$work&displaytype=x
+ml",
            "c:/perlmonks/$work.xml"
        );

        $Qresults->enqueue( "$work returned $rc" );
    }
    return;
}

my @t = map{ threads->new( \&work ) } 1 .. $THREADS;

open IN, '<', $ARGV[ 0 ] or die "$ARGV[ 0 ] : $!";

my $cResults = 0;
while( <IN> ) {
    chomp;
    $Qwork->enqueue( $_ ) and $cResults++;
    sleep 1 while $Qwork->pending > 100;
}

$_->join for @t;

my %summary;
$Qresults->dequeue =~ m[(\S+) returned (\S+)]
    and push @{ $summary{ $2 } }, $1
    while $Qresults->pending;

for my $key ( sort keys %summary ) {
    print $key, scalar @{ $summary{ $key } };
}

printf 'Enter to see details or ^C'; <STDIN>;

for my $key ( sort keys %summary ) {
    print $key;
    my $details =  join ' ', @{ $summary{ $key } };
    $details =~ s[.{1,80} ][\n]g;
    print $details;
}
[download]