#!c:/usr/bin/perl

# CPAN modules
use Thread::Pool;
use LWP::UserAgent;
use HTML::LinkExtor;
use URI::URL;

$pool = Thread::Pool->new(
	{
		optimize => 'cpu',          # default: memory
		do => 'XXX',        # must have
		checkpoint => \&checkpoint,
		frequency => 1000,
		workers => 5,     # default: 1
		maxjobs => 50,     # default: 5 * workers
		minjobs => 5,      # default: maxjobs / 2
	},
);

# Start up some threads
$pool->job( "http://www.yahoo.com" ); 
$pool->job( "http://www.eroticfalconry.com" );
$pool->join;          # wait for all threads to finish

#### subroutines

sub XXX {
	$current_url = @_[0];
	@queue = ();
	
	while( $current_url ne "" ){
		@links = get("$current_url" );
		print "links in url\n********************\n";
		print join("\n", @links), "\n";
		push(@queue,@links);
		#print "links in queue\n********************\n";
		#print join("\n", @queue), "\n";
		$current_url = pop(@queue);
		print ">>>>new url=$current_url\n"; #this is where we go next
	}
}

my $p = HTML::LinkExtor->new(\&callback);
my @links = ();

# Set up a callback that collect links
sub callback {
   my($tag, %attr) = @_;
   return if $tag ne 'a';  # we only look closer at <a ...>
   push(@links, values %attr);
}

sub get {
  $url = "@_[0]";  
  print "Entering html::getlinks, url=$url\n";
  my $ua = LWP::UserAgent->new;
	$res = $ua->request(HTTP::Request->new(GET => $url), sub {$p->parse($_[0])});
	$base = $res->base; # Expand all URLs to absolute ones
	@links = map { $_ = url($_, $base)->abs; } @links;
	return @links;
}

$version = 1;