in reply to Return all the data from child to parent with Parallel::Forkmanager

Hi Microcebus,

You were close. Notice the arguments given to $pm->start and $pm->finish. Start accepts an optional identifier for the process. That same identifier is used as the 3rd argument for the on_finish callback. In the demonstration that follows, I puposely have read_GTF die with an exception. The output is the same whether threads equals 1 or 4.

Like afoken mentioned here, Parallel::ForkManager must be 0.7.6 or later for the demonstration to run.

use strict;
use warnings;
use feature 'say';

use Parallel::ForkManager;

my $threads = 4;

my @name = ('read_genome', 'read_mapfile', 'read_GTF', 'read_RM');
my @task = (\&read_genome, \&read_mapfile, \&read_GTF, \&read_RM);
my (@ret, @err);

if ($threads == 1) {
    for my $id (0 .. $#task) {
        $ret[$id] = eval { $task[$id]->() };
        $err[$id] = $@;
    }
}
else {
    my $pm = new Parallel::ForkManager($threads);
    $pm->set_waitpid_blocking_sleep(0);

    $pm->run_on_finish( sub {
        my ($pid, $exit, $id, $signal, $core, $data) = @_;
        $ret[$id] = delete $data->{ret};
        $err[$id] = delete $data->{err};
    });

    for my $id (0 .. $#task) {
        $pm->start($id) and next;
        my $res = eval { $task[$id]->() };
        $pm->finish(0, { ret => $res, err => $@ });
    }

    $pm->wait_all_children;
}

sub read_genome {
    # do something
    return { 'aa' => 'bb' };
}
sub read_mapfile {
    # do something
    return { 'cc' => 'dd' };
}
sub read_GTF {
    # do something
    die 'exception';
    return { 'ee' => 'ff' }; # not reached
}
sub read_RM {
    # do something
    return { 'gg' => 'hh' };
}

# use data generated in the subroutines

use Data::Dumper;

for my $id (0 .. $#task) {
    say "## ", $name[$id];
    if (length $err[$id]) {
        say "ERROR: ", $err[$id];
        next;
    }
    say Dumper($ret[$id]);
}

[download]

Output:

## read_genome
$VAR1 = {
          'aa' => 'bb'
        };

## read_mapfile
$VAR1 = {
          'cc' => 'dd'
        };

## read_GTF
ERROR: exception at j1.pl line 54.

## read_RM
$VAR1 = {
          'gg' => 'hh'
        };

[download]

Regards, Mario

Replies are listed 'Best First'.
Re^2: Return all the data from child to parent with Parallel::Forkmanager
by marioroy (Prior) on Aug 19, 2017 at 04:24 UTC

    Hi again,

    The following provides a MCE::Loop demonstration. This time, am using a hash for storing the returned data. Workers pass data using socket handles inside MCE. Thus, no temp files are made for sending data to the parent process.

    use strict;
use warnings;
use feature 'say';

use MCE::Loop;

my $threads = 4;

my @name = ('read_genome', 'read_mapfile', 'read_GTF', 'read_RM');
my @task = (\&read_genome, \&read_mapfile, \&read_GTF, \&read_RM);
my %ret;

if ($threads == 1) {
    for my $id (0 .. $#task) {
        $ret{"$id"} = eval { $task[$id]->() };
        $ret{"$id:err"} = $@;
    }
}
else {
    MCE::Loop->init(
        max_workers => $threads,
        posix_exit  => 1,
        chunk_size  => 1
    );
    %ret = mce_loop {
        my $id  = $_;
        my $res = eval { $task[$id]->() };
        MCE->gather( $id => $res, "$id:err" => $@ );
    }
    [ 0 .. $#task ];

    MCE::Loop->finish;
}

sub read_genome {
    # do something
    return { 'aa' => 'bb' };
}
sub read_mapfile {
    # do something
    return { 'cc' => 'dd' };
}
sub read_GTF {
    # do something
    die 'exception';
    return { 'ee' => 'ff' }; # not reached
}
sub read_RM {
    # do something
    return { 'gg' => 'hh' };
}

# use data generated in the subroutines

use Data::Dumper;

for my $id (0 .. $#task) {
    say "## ", $name[$id];
    if (length $ret{"$id:err"}) {
        say "ERROR: ", $ret{"$id:err"};
        next;
    }
    say Dumper($ret{$id});
}

    [download]

    Output:

    ## read_genome
$VAR1 = {
          'aa' => 'bb'
        };

## read_mapfile
$VAR1 = {
          'cc' => 'dd'
        };

## read_GTF
ERROR: exception at j2.pl line 45, <__ANONIO__> line 2.

## read_RM
$VAR1 = {
          'gg' => 'hh'
        };

    [download]

    Regards, Mario

[reply]
[d/l]
[select]

In Section Seekers of Perl Wisdom