If you have the time and inclination, try this and see how you fare. You might have to make a few tweaks, it compiles clean but is otherwise untested beyond a mental run through, which given my mind is notoriously unreliable:)
sub mergeLogs { my ($day, @files) = @_; my @lines; foreach my $file (@files) { my $fh = openLogFile($file); warn "$0: ignoring file $file\n", next unless defined $fh; warn "-> processing $file\n" if $opts{'verbose'} > 0; while( <$fh> ) { next unless /Running|Dump|FromCB|Update/o; next if exists $opts{'day'} && ! /^$opts{'day'}/o; next if exists $opts{'user'} && ! /[\(\[]\s*(?:$opts{'user +'})/o; $opts{'server'} = lc $1, next if !exists $opts{'server'} && /\* Running on (\w+) -/; my $time = /(\d{2}:\d{2}:\d{2}\.\d{3})/o; next if exists $opts{'start-time'} && $time lt $opts{'star +t-time'}; next if exists $opts{'stop-time'} && $time gt $opts{'stop +-time'}; s/ {2,}/ /go; s/ ?: / /go; s/^((?:\S+ ){3}).+?\[?I\]?:/$1/o; s/ ACK (\w) / $1 ACK /o; warn $_ if $opts{'verbose'} > 3; ## prepend the time key now save reparsing later push @lines, $time . $_; } close $fh; } ## Sort in-place to save memory shuffling ## (if it decides to play ball today) @lines = sort @lines; substr $_, 0, 12, '' for @lines; ## Trim the prepended keys return \@lines; }
In reply to Re^3: How to improve speed of reading big files
by BrowserUk
in thread How to improve speed of reading big files
by korlaz
| For: | Use: | ||
| & | & | ||
| < | < | ||
| > | > | ||
| [ | [ | ||
| ] | ] |