in reply to Re^12: PDL and srand puzzle - one billion demonstration (updated)
in thread PDL and srand puzzle
I tried another variation for PDL.
Update: Reduce RAM usage by processing a sequence of numbers instead (~ 45MB per worker). For predictable output, this requires seeding the generator per each sequence. Previously, the code spawned 100 workers and 1e7 loop iterations (~ 24GB RAM).
use v5.030; use PDL; use MCE 1.895; CORE::srand(42); # This also, for MCE predictable results # MCE sets internal seed = CORE::random() PDL::srandom(42); # PDL::srand(N) v2.062 ~ v2.089 MCE->new( max_workers => MCE::Util::get_ncpu(), chunk_size => 1, init_relay => 0, posix_exit => 1, sequence => [ 1, 1000 ], user_func => sub { # my ($mce, $seq, $chunk_id) = @_; my $seq = $_; my $output = ""; # Worker seeds generator using MCE's seed and wid value. # For sequence of numbers, compute similarly using $seq value. my $seed = abs(MCE->seed - ($seq * 100000)) % 1073741780; # PDL::srand($seed); # PDL v1.062 ~ 1.089 PDL::srandom($seed); # PDL v1.089_01 my $pdl = PDL->random(1e6); foreach (0 .. $pdl->nelem - 1) { my $r = $pdl->at($_); $output .= "$r\n"; } MCE::relay { print $output; }; } )->run;
Time to run:
$ time perl test_pdl.pl > out # 17GB file size PDL->random . . . . . . . . . . . 27.211s $ wc -l out 1000000000
|
|---|