Greetings, biohisham,

Gathering nested data is possible with a gather callback. MCE::Relay is used to relay the index line counter. I commented out the section regarding past index line. Starting from 0 each time made me wonder. I changed it to relay index line.

Please note that $ret_FullGlobal, $ret_ProbState, and $ret_ErrProbState hashes are HoH-H and HoH. Important! Remember to "not" gather HoH->A and HoA structures containing "only" high indexes. Perl is likely to reserve memory for the empty array slots. For your use case, HoH-H and HoH are your friends with regards to gathering nested structure. The gather callback can then append/populate the global HoH->A and HoA structures.

use strict;
use warnings;
use Data::Dumper;
use MCE::Loop;

no warnings 'uninitialized';

my $ProbTrajfile = "states_example.csv";
my $ProbTrajtable_file = "states_example_table.csv";

open(my $ProbTraj, "<", $ProbTrajfile) or die;
my $line = <$ProbTraj>;
my @firstlinelist = split(/\t/, $line);
my $name;
my $IndexProba = 4;
my @firstline_woutH = @firstlinelist[4..$#firstlinelist];

foreach $name (@firstline_woutH)
{
    $_ = $name;
    last if (! m/HD=/);
    $IndexProba++;
}

my %FullGlobal;
my %ProbState;
my %ErrProbState;
my $IndexLine = 0;

$line = <$ProbTraj>; chomp($line);
my @linelist = split(/\t/, $line);

$FullGlobal{Time}[$IndexLine] = $linelist[0];
$FullGlobal{TH}[$IndexLine] = $linelist[1];
$FullGlobal{ErrTH}[$IndexLine] = $linelist[2];
$FullGlobal{H}[$IndexLine] = $linelist[3];

#create probState hash
for (my $i = $IndexProba; $i <= $#linelist; $i = $i + 3)
{
    my $State = "[".$linelist[$i]."]";
    $ProbState{$State}[$IndexLine] = $linelist[$i+1];
    $ErrProbState{$State}[$IndexLine] = $linelist[$i+2];
}

sub gather_callback {
    my ($ret_FullGlobal, $ret_ProbState, $ret_ErrProbState) = @_;

    foreach my $i (keys %{ $ret_FullGlobal->{Time} })
    {
        $FullGlobal{Time}[$i]  = $ret_FullGlobal->{Time}{$i};
        $FullGlobal{TH}[$i]    = $ret_FullGlobal->{TH}{$i};
        $FullGlobal{ErrTH}[$i] = $ret_FullGlobal->{ErrTH}{$i};
        $FullGlobal{H}[$i]     = $ret_FullGlobal->{H}{$i};
    }

    foreach my $State (keys %{ $ret_ProbState })
    {
        while (my ($i,$v) = each %{ $ret_ProbState->{$State} })
        {
            $ProbState{$State}[$i] = $v;
        }
    }

    foreach my $State (keys %{ $ret_ErrProbState })
    {
        while (my ($i,$v) = each %{ $ret_ErrProbState->{$State} })
        {
            $ErrProbState{$State}[$i] = $v;
        }
    }
}

MCE::Loop::init {
    use_slurpio => 1,
    chunk_size  => '20k',
    max_workers => 16,
    init_relay  => 0,
    gather      => \&gather_callback,
};

mce_loop_f {
    my ($mce, $chunk_ref, $chunk_id) = @_;

    my $ret_FullGlobal   = {};
    my $ret_ProbState    = {};
    my $ret_ErrProbState = {};

    my $numlines = ${ $chunk_ref } =~ tr/\n//;
    my $RelayIndexLine = MCE->relay_recv;
    MCE::relay { $_ += $numlines };

    my $IndexLine = $RelayIndexLine;
    open my $ifh, "<", $chunk_ref;

    while (<$ifh>)
    {   
        chomp($_);
        @linelist = split(/\t/, $_);

        $IndexLine++;
        #print $IndexLine, $/;

        $ret_FullGlobal->{Time}{$IndexLine}  = $linelist[0];
        $ret_FullGlobal->{TH}{$IndexLine}    = $linelist[1];
        $ret_FullGlobal->{ErrTH}{$IndexLine} = $linelist[2];
        $ret_FullGlobal->{H}{$IndexLine}     = $linelist[3];   

        foreach my $State (keys(%ProbState))
        {
            $ret_ProbState->{$State}{$IndexLine} = 0;
            $ret_ErrProbState->{$State}{$IndexLine} = 0;
        }

        for (my $i=$IndexProba; $i<=$#linelist; $i=$i+3)
        {
            my $State = "[".$linelist[$i]."]";
            if (defined $ProbState{$State}[$IndexLine-1])
            {
                $ret_ProbState->{$State}{$IndexLine} = $linelist[$i+1]
+;
                $ret_ErrProbState->{$State}{$IndexLine} = $linelist[$i
++2];
            }
            else
            {
              # for (my $PastIndexLine=$RelayIndexLine; $PastIndexLine
+<$IndexLine; $PastIndexLine++)
              # {
              #     $ret_ProbState->{$State}{$PastIndexLine} = 0;
              #     $ret_ErrProbState->{$State}{$PastIndexLine} = 0;
              # }
                $ret_ProbState->{$State}{$IndexLine} = $linelist[$i+1]
+;
                $ret_ErrProbState->{$State}{$IndexLine} = $linelist[$i
++2];
            }
        }
    }
    close $ifh;

    MCE->gather($ret_FullGlobal, $ret_ProbState, $ret_ErrProbState);

} $ProbTrajfile;

MCE::Loop->finish;

#print "FullGlobal: ", Dumper(\%FullGlobal), $/;
#print "ProbState: ", Dumper(\%ProbState), $/;
#print "ErrProbState: ", Dumper(\%ErrProbState), $/;

#write the trajectory table in file (the hashes ain't accessible outsi
+de mce_loop_f
my @StateList = keys(%ProbState);

open (my $ProbTrajTable,">", $ProbTrajtable_file) or die ($!);
print $ProbTrajTable "Time\t";
print $ProbTrajTable "TH\t";
print $ProbTrajTable "ErrTH\t";
print $ProbTrajTable "H\t" ;
print $ProbTrajTable "\n";

foreach my $State (@StateList)
{
    $_ = $State;
    s/ //g;
    print $ProbTrajTable "Prob".$_."\t"."ErrProb".$_."\t";
}

print $ProbTrajTable "\n";

for (my $i=0; $i<=38; $i++)
{
    print $ProbTrajTable $FullGlobal{Time}[$i]."\t";
    print $ProbTrajTable $FullGlobal{TH}[$i]."\t";
    print $ProbTrajTable $FullGlobal{ErrTH}[$i]."\t";
    print $ProbTrajTable $FullGlobal{H}[$i]."\t";
    foreach my $State (@StateList)
    {
        print $ProbTrajTable $ProbState{$State}[$i]."\t".$ErrProbState
+{$State}[$i]."\t";
    }
    print $ProbTrajTable "\n";
}

close($ProbTrajTable);

[download]

In reply to Re^5: MCE: How to access variables globally by marioroy
in thread MCE: How to access variables globally by biohisham

Title:
Use:  <p> text here (a paragraph) </p>
and:  <code> code here </code>
to format your post, it's "PerlMonks-approved HTML":


  • Posts are HTML formatted. Put <p> </p> tags around your paragraphs. Put <code> </code> tags around your code and data!
  • Titles consisting of a single word are discouraged, and in most cases are disallowed outright.
  • Read Where should I post X? if you're not absolutely sure you're posting in the right place.
  • Please read these before you post! —
  • Posts may use any of the Perl Monks Approved HTML tags:
    a, abbr, b, big, blockquote, br, caption, center, col, colgroup, dd, del, details, div, dl, dt, em, font, h1, h2, h3, h4, h5, h6, hr, i, ins, li, ol, p, pre, readmore, small, span, spoiler, strike, strong, sub, summary, sup, table, tbody, td, tfoot, th, thead, tr, tt, u, ul, wbr
  • You may need to use entities for some characters, as follows. (Exception: Within code tags, you can put the characters literally.)
            For:     Use:
    & &amp;
    < &lt;
    > &gt;
    [ &#91;
    ] &#93;
  • Link using PerlMonks shortcuts! What shortcuts can I use for linking?
  • See Writeup Formatting Tips and other pages linked from there for more info.