in reply to How to use threads to write worksheets of excel

Update: Added missing line after looping each region.

Update: This example runs nearly as fast as serial code and completes in 14.5 seconds without threads.

The following demo demonstrates multiple workers with a single writer running simultaneously. Locking between workers is handled automatically by MCE.

use strict; use warnings; use Excel::Writer::XLSX; use MCE::Loop Sereal => 1; my $nodeList = [ [ 'AMS' , 'a' ], [ 'APJ' , 'ap' ], [ 'EMEA', 'e' ], ]; my ($workbook, %worksheets, $format); $workbook = Excel::Writer::XLSX->new("Node_List.xlsx"); $workbook->set_properties( title => 'Node List', author => 'L_WC demo', comments => 'Node List', ); $format = $workbook->add_format(bg_color => 44); $format->set_align('center'); foreach (@{ $nodeList }) { $worksheets{ $_->[0] } = $workbook->add_worksheet( $_->[0] ); } MCE::Loop::init( chunk_size => 1, max_workers => scalar(@{ $nodeList }), gather => sub { my $region = shift; while (@{ $_[0] }) { my @args = splice(@{ $_[0] }, 0, 3); $worksheets{$region}->write(@args, $format) } }, ); mce_loop { doSomething($_->[0], $_->[1]) } $nodeList; $workbook->close(); print "Node List is Done.\n"; sub doSomething { my ($region, $sql) = @_; if ($region eq 'AMS') { my @data; my $n = $region . '_'; for (0..65534) { push @data, $_, 0, $n . $_; push @data, $_, 1, $_ + 4; push @data, $_, 2, $_ + 3; push @data, $_, 3, $_ + 2; push @data, $_, 4, $_ + 1; if ($_ % 4000 == 0) { MCE->gather($region, \@data); @data = (); } } MCE->gather($region, \@data) if @data; print "AMS -----DONE.\n"; } elsif ($region eq 'APJ') { my @data; my $n = $region . '_'; for (0..65534) { push @data, $_, 0, $n . $_; push @data, $_, 1, $_ + 1; push @data, $_, 2, $_ + 3; push @data, $_, 3, $_ + 4; push @data, $_, 4, $_ + 2; if ($_ % 4000 == 0) { MCE->gather($region, \@data); @data = (); } } MCE->gather($region, \@data) if @data; print "APJ -----DONE.\n"; } elsif ($region eq 'EMEA') { my @data; my $n = $region . '_'; for (0..65534) { push @data, $_, 0, $n . $_; push @data, $_, 1, $_ + 1; push @data, $_, 2, $_ + 2; push @data, $_, 3, $_ + 3; push @data, $_, 4, $_ + 4; if ($_ % 4000 == 0) { MCE->gather($region, \@data); @data = (); } } MCE->gather($region, \@data) if @data; print "EMEA -----DONE.\n"; } return; }

Kind regards, Mario

Replies are listed 'Best First'.
Re^2: How to use threads to write worksheets of excel
by L_WC (Initiate) on Jul 01, 2015 at 02:18 UTC
    hi, Mario thanks for your replay. And i will check out your code. and i have update my question. could you give me more ideas about database base's tables import to worksheets and performance? thanks very much.
        did you have a try DBI and MCE to process on a case? is MCE shared by the same workbook?