sub walk { my ( $job, $result, $failed ) = @_; # Yielding is critical when running an event loop in parallel. # Not doing so means that the app may reach contention points # with the firewall and likely impose unnecessary hardship at # the OS level. The idea here is not to have multiple workers # initiate HTTP requests to a batch of URLs at the same time. # Yielding in 1.827+ behaves more like scatter for the worker # to run solo in a fraction of time. MCE::Hobo->yield( 0.03 ); # MCE::Hobo 1.827 my $cv = AnyEvent->condvar(); # Populate the hash ref for URLs it could reach. # Do not mix AnyEvent timeout and Hobo timeout. # Choose to do the event timeout if available. foreach my $url ( @{ $job->{INPUT} } ) { $cv->begin(); http_get $url, timeout => 2, sub { my ( $data, $headers ) = @_; $result->{$url} = $data; $cv->end(); }; } $cv->recv(); # Populate the array ref for URLs it could not reach. foreach my $url ( @{ $job->{INPUT} } ) { push @{ $failed }, $url unless (exists $result->{ $url }); } return; }