#! perl -slw
use strict;
use Time::HiRes qw[ sleep ];
use threads;
use threads::shared;

my @commands = map {
    my $t = 2+int( rand 5 );
    qq[ perl -E"sleep 1 while ++\$n < $t" ];
} 1 .. 300;


my $running :shared = 0;
my $done :shared = 0;
for my $cmd ( @commands ) {
    async{
        ++$running;
        system $cmd;
        --$running;
        ++$done;
    }->detach;
    printf( "\r$done, $running" ), sleep 0.1
        while $running > 2;
}
[download]

Search CPAN.   There are many comprehensive workload management systems available, such as POE.

You will not have to “reinvent this wheel.”

Could you show me how to modify this to provide a tasks completed progress indicator per the example above?

#!/usr/bin/perl
use warnings;
use strict;
use POE qw(Wheel::Run Filter::Reference);
sub MAX_CONCURRENT_TASKS () { 3 }
my @tasks = qw(one two three four five six seven eight nine ten);

POE::Session->create(
  inline_states => {
    _start      => \&start_tasks,
    next_task   => \&start_tasks,
    task_result => \&handle_task_result,
    task_done   => \&handle_task_done,
    task_debug  => \&handle_task_debug,
    sig_child   => \&sig_child,
  }
);

sub start_tasks {
  my ($kernel, $heap) = @_[KERNEL, HEAP];
  while (keys(%{$heap->{task}}) < MAX_CONCURRENT_TASKS) {
    my $next_task = shift @tasks;
    last unless defined $next_task;
    print "Starting task for $next_task...\n";
    my $task = POE::Wheel::Run->new(
      Program      => sub { do_stuff($next_task) },
      StdoutFilter => POE::Filter::Reference->new(),
      StdoutEvent  => "task_result",
      StderrEvent  => "task_debug",
      CloseEvent   => "task_done",
    );
    $heap->{task}->{$task->ID} = $task;
    $kernel->sig_child($task->PID, "sig_child");
  }
}

sub do_stuff {
  binmode(STDOUT);    # Required for this to work on MSWin32
  my $task   = shift;
  my $filter = POE::Filter::Reference->new();

  sleep(rand 5);

  my %result = (
    task   => $task,
    status => "seems ok to me",
  );

  my $output = $filter->put([\%result]);
  print @$output;
}

sub handle_task_result {
  my $result = $_[ARG0];
  print "Result for $result->{task}: $result->{status}\n";
}

sub handle_task_debug {
  my $result = $_[ARG0];
  print "Debug: $result\n";
}

sub handle_task_done {
  my ($kernel, $heap, $task_id) = @_[KERNEL, HEAP, ARG0];
  delete $heap->{task}->{$task_id};
  $kernel->yield("next_task");
}

sub sig_child {
  my ($heap, $sig, $pid, $exit_val) = @_[HEAP, ARG0, ARG1, ARG2];
  my $details = delete $heap->{$pid};

  # warn "$$: Child $pid exited";
}

$poe_kernel->run();
exit 0;
[download]

If all your tasks can be identified by a simple ID, and you are running on Linux, you can "abuse" existing tools:

$ perl generate_ids.pl | xargs -P 3 -n 1 perl process_single_id.pl
[download]