LWP::Protocol::collect: read 1418 bytes
LWP::Protocol::collect: read 1418 bytes
LWP::Protocol::collect: read 1418 bytes
####
my $pm = Parallel::ForkManager->new(10);
####
fetch_content(@urls);
$pm->wait_all_children;
## Run when children are forked
$pm->run_on_start(
sub { my ($pid, $link) = @_;
$link =~ s/\s+$//;
## Count the pages fetched thus far
$pagecount++;
}
);
## Run when child processes complete
$pm->run_on_finish(
sub { my ($pid, $exit_code, $ident) = @_;
print "\n** $ident out of the pool ".
"with PID $pid and exit code:
$exit_code\n";
}
);
## Run when blocking/waiting for children
$pm->run_on_wait(
sub {
print "-"x74, "\n";
print "\n** Waiting for child ...\n";
print "-"x74, "\n";
},
0.1
);
## Fetch the actual page and links
sub fetch_content {
my @urls = @_;
for my $link (@urls) {
my $pid = $pm->start($link) and next;
# fetch the page, extract the links
# (all of the fetching/extraction works)
$pm->finish;
}
}