use strict; use warnings; use feature qw(say); use MCE::Loop chunk_size => 1, max_workers => 4; use MCE::Shared; my @files = glob '*.gz'; my $result = MCE::Shared->hash(); mce_loop { my ($mce, $chunk_ref, $chunk_id) = @_; ## $file = $_; same thing when chunk_size => 1 my $file = $chunk_ref->[0]; ## http://www.zlib.net/pigz/ ## For pigz, we want -p1 to run on one core only. ## open my $fh, '-|', 'pigz', '-dc', '-p1', $file or do { ... } open my $fh, '-|', 'gzip', '-dc', $file or do { warn "open error ($file): $!\n"; MCE->next(); }; my $count = 0; while ( my $line = <$fh> ) { $count++; # simulate filtering or processing } close $fh; ## Send output to the manager process. ## Ensures workers do not garble STDOUT. MCE->say("$file: $count lines"); ## Store key-value pair. $result->set($file, $count); } @files; ## Workers may persist after running. Request workers to exit. MCE::Loop->finish(); ## Ditto, same output using shared data. for my $file (@files) { say "$file: ", $result->get($file), " lines"; }