#!/usr/bin/env perl
use warnings;
use strict;
use Path::Class qw/dir/;
use IO::Compress::Gzip qw/Z_BEST_COMPRESSION/;
use Text::CSV; # also install Text::CSV_XS for speed
my $VERBOSE = 1;
# Note: Currently using a fixed pattern for the output file names
die "Usage: $0 DIR LINES\n" unless @ARGV==2;
my $DIR = dir($ARGV[0]);
die "Bad DIR '$DIR'" unless -d $DIR;
my $LINES = $ARGV[1];
$LINES =~ /\A(?!0)[0-9]+\z/ or die "Bad LINES '$LINES'\n";
my @files = map { $$_[0] } sort { $$a[1]<=>$$b[1] } map { [$_,-s $_] }
grep { !$_->is_dir && $_->basename=~/\.csv\z/i } $DIR->children;
my $csv = Text::CSV->new({ binary=>1, auto_diag=>2, eol=>$/ });
my ($ofcnt,$ofln,$ofh) = (0,0);
for my $infile (@files) {
print STDERR "Reading $infile\n" if $VERBOSE;
my $ifh = $infile->openr;
$csv->getline($ifh); # drop first line
while ( my $row = $csv->getline($ifh) ) {
if (!defined $ofh) {
my $outfile = $DIR->file("part-".(++$ofcnt).".csv.gz");
if ( -e $outfile )
{ warn "Warning: Overwriting $outfile\n" }
else { print STDERR "Writing $outfile\n" if $VERBOSE }
$ofh = IO::Compress::Gzip->new( $outfile->openw,
Level => Z_BEST_COMPRESSION )
or die "$outfile: $IO::Compress::Gzip::GzipError\n";
$ofln = 0;
}
$csv->print($ofh, $row);
if ( ++$ofln >= $LINES ) { $ofh->close; $ofh=undef }
}
$csv->eof or $csv->error_diag;
}
$ofh->close if defined $ofh;
-
Are you posting in the right place? Check out Where do I post X? to know for sure.
-
Posts may use any of the Perl Monks Approved HTML tags. Currently these include the following:
<code> <a> <b> <big>
<blockquote> <br /> <dd>
<dl> <dt> <em> <font>
<h1> <h2> <h3> <h4>
<h5> <h6> <hr /> <i>
<li> <nbsp> <ol> <p>
<small> <strike> <strong>
<sub> <sup> <table>
<td> <th> <tr> <tt>
<u> <ul>
-
Snippets of code should be wrapped in
<code> tags not
<pre> tags. In fact, <pre>
tags should generally be avoided. If they must
be used, extreme care should be
taken to ensure that their contents do not
have long lines (<70 chars), in order to prevent
horizontal scrolling (and possible janitor
intervention).
-
Want more info? How to link
or How to display code and escape characters
are good places to start.
|