in reply to Re^8: Parallel::ForkManager is time consuming...takes too long
in thread Parallel::ForkManager is time consuming...takes too long

The code had still some bugs. Now it works.
#!/usr/bin/perl @array_a = (66.93, 66.91, 66.84, 66.82, 66.87, 66.91, 66.7, 66.8, 66.5 +7, 66.61, 66.81, 66.72, 66.78, 66.76, 66.86, 66.73, 66.82, 66.95, 66. +88, 66.85, 66.87, 66.75, 66.88, 66.93, 66.94, 66.87, 66.88, 66.85, 66 +.83, 66.89, 66.96, 67.03, 67.08, 67.22, 67.36, 67.55, 67.5, 67.49, 67 +.38, 67.49, 67.47, 67.7, 67.62, 67.58, 67.61, 67.79, 67.73, 67.79, 67 +.81, 67.94, 67.99, 68.18, 68.01, 67.82, 67.71, 67.71, 67.65, 67.69, 6 +7.7, 67.7, 67.69, 67.66, 67.62, 67.58, 67.56, 67.5, 67.24, 67.36, 67. +45, 67.46, 67.38, 67.55, 67.53, 67.56, 67.67, 67.78, 67.78, 67.78, 67 +.78, 67.84, 67.78, 67.74, 67.9, 67.72, 67.74, 67.79, 67.86, 67.88, 67 +.93, 67.97, 68.09, 67.76, 67.89, 67.81, 67.97, 68, 67.84, 67.95, 67.9 +7, 68.02, 68.07, 68.03, 68.08, 68.05, 68.05, 68.07, 68.14, 68.18, 68. +21, 68.13, 68.07, 68.02, 68, 67.96); @array_b = (10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 9, 9, 9, 9 +, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, +10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 9, 9, 9, 9, 9, 9, 8, 8, 8 +, 8, 8, 8, 7, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7 +, 8, 8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, +10, ); my ($array_ref_c, $array_ref_d, $array_ref_e) = subroutine(\@array_a,\ +@array_b); my @array_c = @$array_ref_c; my @array_d = @$array_ref_d; my @array_e = @$array_ref_e; foreach my $i (0 .. $#array_a) { print($i, "\t",$array_b[$i],"\t",$array_a[$i],"\t",$array_c[$i],"\t" +,$array_d[$i],"\t",$array_e[$i],"\n"); } sub subroutine { my ($listaref, $listbref) = @_; my (@array_a) = @$listaref; my (@array_b) = @$listbref; foreach my $i (10 .. $#array_a) { $array_c[$i] = 0; $array_d[$i] = 0; $array_e[$i] = 0; if (( $array_b[$i] >= 5 ) && ( $array_b[$i] < 7 )) { for ( $l = 0 ; $l < $array_b[$i] ; $l++ ) { $array_c[$i] = $array_c[$i] + $array_a[$i - $l]; } $array_c[$i] = $array_c[$i] / $array_b[$i]; } if (( $array_b[$i] >= 7 ) && ( $array_b[$i] < 9 )) { for ( $l = 0 ; $l < $array_b[$i] ; $l++ ) { $array_d[$i]= $array_d[$i] + $array_a[$i - $l]; } $array_d[$i] = $array_d[$i] / $array_b[$i]; } if (( $array_b[$i] >= 9 ) && ( $array_b[$i] < 15 )) { for ( $l = 0 ; $l < $array_b[$i] ; $l++ ) { $array_e[$i]= $array_e[$i] + $array_a[$i - $l]; } $array_e[$i] = $array_e[$i] / $array_b[$i]; } #print($i, "\t",$array_b[$i],"\t",$array_a[$i],"\t",$array_c[$i],"\t +",$array_d[$i],"\t",$array_e[$i],"\n"); } return (\@array_c, \@array_d, \@array_e); } __END__

Replies are listed 'Best First'.
Re^10: Parallel::ForkManager is time consuming...takes too long
by BrowserUk (Patriarch) on Aug 16, 2011 at 19:10 UTC

    I just ran the above code with arrays A & B containing 100,000 values each. It completed in a little over half a second

    C:\test>920533 >920533.obig Took 0.609000 seconds C:\test>wc -l 920533.obig 100000 920533.obig

    What made you think you needed to use multiprocessing to meet your 30 second target time?


    Examine what is said, not who speaks -- Silence betokens consent -- Love the truth but pardon error.
    "Science is about questioning the status quo. Questioning authority".
    In the absence of evidence, opinion is indistinguishable from prejudice.
      I have tonnes of such subroutines and the entire code takes 1 minute and 35 seconds. So when I multiprocess all of them it will reduce the time.
        I have tonnes of such subroutines and the entire code takes 1 minute and 35 seconds.

        Hm. 95 / 0.609 implies that "tonnes" == 156 subroutines?

        If that is really the case, then recoding them all similar to this:

        use List::Util qw[ sum ]; ... sub subroutine { our( @a, @b, @c, @d, @e ); ( *a, *b, *c, *d, *e ) = @_; foreach my $i ( 10 .. $#a ) { $c[$i] = $d[$i] = $e[$i] = 0; my $b = $b[$i]; if( ( $b >= 5 ) && ( $b < 7 ) ) { $c[$i] += sum @a[ $i-$b+1 .. $i ]; $c[$i] /= $b; } elsif( ( $b >= 7 ) && ( $b < 9 ) ) { $d[$i] += sum @a[ $i-$b+1 .. $i ]; $d[$i] /= $b; } elsif( ( $b >= 9 ) && ( $b < 15 ) ) { $e[$i] += sum @a[ $i-$b+1 .. $i ]; $e[$i] /= $b; } } return; } ## called like this my @a = ...; my @b = ...; my( @c, @d, @e ); subroutine( \@a, \@b, \@c, \@d, \@e );

        which runs in 1/3rd the time of your original, should get you close to your target. If not, then you'd have to sort out the inefficiencies in the rest of your code as well.

        If it is really going to be necessary to multi-process this to achieve your target, then you should not be trying to inject the forks or threads into the loops within these subroutines -- as in your OP code-- but rather call the subroutines themselves concurrently. To advise you further on doing that, I'd need to see the calling code in its entirety.


        Examine what is said, not who speaks -- Silence betokens consent -- Love the truth but pardon error.
        "Science is about questioning the status quo. Questioning authority".
        In the absence of evidence, opinion is indistinguishable from prejudice.