#!/usr/bin/env perl
use warnings;
use strict;
use Path::Class qw/dir/;
use IO::Compress::Gzip qw/Z_BEST_COMPRESSION/;
use Text::CSV; # also install Text::CSV_XS for speed

my $VERBOSE = 1;
# Note: Currently using a fixed pattern for the output file names

die "Usage: $0 DIR LINES\n" unless @ARGV==2;
my $DIR = dir($ARGV[0]);
die "Bad DIR '$DIR'" unless -d $DIR;
my $LINES = $ARGV[1];
$LINES =~ /\A(?!0)[0-9]+\z/ or die "Bad LINES '$LINES'\n";

my @files = map { $$_[0] } sort { $$a[1]<=>$$b[1] } map { [$_,-s $_] }
    grep { !$_->is_dir && $_->basename=~/\.csv\z/i } $DIR->children;

my $csv = Text::CSV->new({ binary=>1, auto_diag=>2, eol=>$/ });

my ($ofcnt,$ofln,$ofh) = (0,0);
for my $infile (@files) {
    print STDERR "Reading $infile\n" if $VERBOSE;
    my $ifh = $infile->openr;
    $csv->getline($ifh); # drop first line
    while ( my $row = $csv->getline($ifh) ) {
        if (!defined $ofh) {
            my $outfile = $DIR->file("part-".(++$ofcnt).".csv.gz");
            if ( -e $outfile )
                { warn "Warning: Overwriting $outfile\n" }
            else { print STDERR "Writing $outfile\n" if $VERBOSE }
            $ofh = IO::Compress::Gzip->new( $outfile->openw,
                Level => Z_BEST_COMPRESSION )
                or die "$outfile: $IO::Compress::Gzip::GzipError\n";
            $ofln = 0;
        }
        $csv->print($ofh, $row);
        if ( ++$ofln >= $LINES ) { $ofh->close; $ofh=undef }
    }
    $csv->eof or $csv->error_diag;
}
$ofh->close if defined $ofh;

[download]

In reply to Re: Complex file manipulation challenge by haukex
in thread Complex file manipulation challenge by jdporter

Title:
Use:  <p> text here (a paragraph) </p>
and:  <code> code here </code>
to format your post, it's "PerlMonks-approved HTML":


  • Posts are HTML formatted. Put <p> </p> tags around your paragraphs. Put <code> </code> tags around your code and data!
  • Titles consisting of a single word are discouraged, and in most cases are disallowed outright.
  • Read Where should I post X? if you're not absolutely sure you're posting in the right place.
  • Please read these before you post! —
  • Posts may use any of the Perl Monks Approved HTML tags:
    a, abbr, b, big, blockquote, br, caption, center, col, colgroup, dd, del, details, div, dl, dt, em, font, h1, h2, h3, h4, h5, h6, hr, i, ins, li, ol, p, pre, readmore, small, span, spoiler, strike, strong, sub, summary, sup, table, tbody, td, tfoot, th, thead, tr, tt, u, ul, wbr
  • You may need to use entities for some characters, as follows. (Exception: Within code tags, you can put the characters literally.)
            For:     Use:
    & &amp;
    < &lt;
    > &gt;
    [ &#91;
    ] &#93;
  • Link using PerlMonks shortcuts! What shortcuts can I use for linking?
  • See Writeup Formatting Tips and other pages linked from there for more info.