This was typed up quickly from memory. I haven't tested this code yet, but you get the general idea. I'm also brand new at JavaScript so please don't judge.

'use strict'

var cluster = require('cluster');
var spawn = require('child_process').spawn;
var stream = require('stream');
var glob = require('glob');

var files = glob.sync('/data/logs/*.log.gz');

function Parser() {
    var t = new stream.Transform( { objectMode: true } );
    var transform = t._transform;
    var flush = t._flush;
    var parse = t.parse;
    return t;
}

function transform(chunk, enc, done) {
    var data = chunk.toString();
    //if flush gives us remaining data from last line, prepend to next
+ chunk
    if (this._lastLineData) data = this._lastLineData + data;
    
    var lines = data.split('\n');
    this._lastLineData = lines[lines.length-1];
    
    for (var i = 0; i < lines.length; i++) {
        this.push(parse.lines[i]);
    }
    done();
}

function flush(done) {
    //If remaining data, pass to transform to prepend to next chunk
    if (this._lastLineData) this.push(this._lastLineData);
    this._lastLineData = null
    done()
}

function parse(line) {
    //Split line into fields, or apply regex to filter results
    return line;
}

if (cluster.isMaster) {

var workers = 24; // number of CPUs
var logsPerWorker = workers / files;
var remainder = workers % files;
var logSlice {};
var sliceStart = 0;
var sliceEnd = 0;

    for (var i = 0; i < workers; i++) {
        if (remainder) {
            sliceEnd++;
            remainder--;
        }
        logSlice[i] = files.slice(sliceStart,sliceEnd);
        sliceStart++;
        sliceEnd = sliceStart;
        
        if (logsPerWorker === 0 && remainder === 0) {
            //if no more logs to distribute, break out of loop
            break;
        }
    }

    for (var i = 0; i < workers; i++) {
        //Fork and pass list of logs for child to process
        cluster.fork({logs: logSlice[i]});
    }
} else {
    var liner = Parser();
    var logs = split.env.logs(','); //read logs passed by parent
    for (var i = 0; i < logs.length; i++){
        var child = spawn('zcat', [ logs[i] ]);
        child.stdout.pipe(liner); //pipe this to stdout or to writeStr
+eam to write to a file
    }
}
[download]

---

---

Posts are HTML formatted. Put <p> </p> tags around your paragraphs. Put <code> </code> tags around your code and data!

Titles consisting of a single word are discouraged, and in most cases are disallowed outright.

Read Where should I post X? if you're not absolutely sure you're posting in the right place.

Please read these before you post! —

• How do I compose an effective node title?
• How do I post a question effectively?
• Markup in the Monastery

Posts may use any of the Perl Monks Approved HTML tags:

a, abbr, b, big, blockquote, br, caption, center, col, colgroup, dd, del, details, div, dl, dt, em, font, h1, h2, h3, h4, h5, h6, hr, i, ins, li, ol, p, pre, readmore, small, span, spoiler, strike, strong, sub, summary, sup, table, tbody, td, tfoot, th, thead, tr, tt, u, ul, wbr

You may need to use entities for some characters, as follows. (Exception: Within code tags, you can put the characters literally.)

	For:		Use:
	&		&
	<		&lt;
	>		&gt;
	[		&#91;
	]		&#93;

Link using PerlMonks shortcuts! What shortcuts can I use for linking?

See Writeup Formatting Tips and other pages linked from there for more info.