$ ls -lh very_huge.file -rw-r--r-- 1 mario staff 20G May 24 14:53 very_huge.file ## grep command $ time grep karl very_huge.file nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl real 6m47.048s ( 407 seconds ) user 6m42.372s sys 0m 4.669s ## bin/mce_grep $ time ./MCE-1.608/bin/mce_grep karl very_huge.file nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl real 2m17.003s ( 137 seconds ) user 17m 9.223s sys 0m33.223s ## examples/egrep.pl $ time ./MCE-1.608/examples/egrep.pl karl very_huge.file nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl real 0m26.447s user 0m22.527s sys 0m 8.459s ## MCE::Loop script $ time ./mce_loop_script.pl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl nose cuke karl Took 25.650 seconds real 0m25.764s user 0m42.494s sys 0m 7.264s #### use MCE::Loop; use Time::HiRes qw( time ); MCE::Loop::init( { max_workers => 4, use_slurpio => 1 } ); my $start = time; my $pattern = 'karl'; my @result = mce_loop_f { my ($mce, $slurp_ref, $chunk_id) = @_; ## Quickly determine if a match is found. ## Basically, only process slurped chunk if true. if ($$slurp_ref =~ /$pattern/im) { my @matches; open my $MEM_FH, '<', $slurp_ref; binmode $MEM_FH, ':raw'; while (<$MEM_FH>) { push @matches, $_ if (/$pattern/); } close $MEM_FH; MCE->gather(@matches); } } 'very_huge.file'; print join('', @result); printf "Took %.3f seconds\n", time - $start;