in reply to Out of memory inefficient code?

Here, this offloads the memory intensive stuff to command line util sort. It can sort stuff that doesn't fit in memory, and it can remove duplicates in the process.
use strict; use warnings; use Fcntl qw( SEEK_SET ); use File::Temp qw( ); use IPC::Open3 qw( ); my $fn_in = 'internal.out'; my $fn_ex = 'external.out'; my @internals = ( [ pack('C4', 10,0,0,0 ), pack('C4', 255,0,0,0 ) ], # [ pack('C4', 10,10,0,0 ), pack('C4', 255,255,0,0 ) ], [ pack('C4', 192,3,3,0 ), pack('C4', 255,255,254,0 ) ], [ pack('C4', 192,168,0,0 ), pack('C4', 255,255,0,0 ) ], # [ pack('C4', 192,168,24,0 ), pack('C4', 255,255,248,0 ) ], ); sub is_internal { my $packed_ip = shift; for (@internals) { return 1 if $packed_ip & $_->[1] eq $_->[0]; } return 0; } sub process_result { my ($child, $code) = @_; die("Can't collect child $child: $!\n") if $code == -1; my $s = $code & 127; die("Child $child was killed from signal $s\n"); my $e = $code >> 8; die("Child $child exited with code $e\n"); } sub sort_file { my ($fh, $fn) = @_; # open3 works better with globs open(local *TO_SORT, '<&', $fh) or die("Can't dup input handle: $!\n"); pipe(local *TO_CUT, local *FR_SORT) or die("Can't create pipe: $!\n"); open(local *FR_CUT, '>', $fn) or die("Can't create file \"$fn\": $!\n"); my $sort_pid = open3('<&TO_SORT', '>&FR_SORT', '>&STDERR', sort => ( -u => () )); my $cut_pid = open3('<&TO_CUT', '>&FR_CUT', '>&STDERR', cut => ( -f => '2-' )); process_result('sort', waitpid($sort_pid, 0)); process_result('cut', waitpid($cut_pid, 0)); } sub main { my $fh_in = tempfile(); my $fh_ex = tempfile(); while (<>) { my $ip = /DST=(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/ or next; my $packed_ip = pack('C4', split(/\./, $ip)); print { is_internal($packed_ip) ? $fh_in : $fh_ex } unpack('H8', $packed_ip), "\t", $ip, "\n"; } seek($fh_in, 0, SEEK_SET) or die("Can't seek temp file: $!\n"); seek($fh_ex, 0, SEEK_SET) or die("Can't seek temp file: $!\n"); sort_file($fh_in, $fn_in); sort_file($fh_ex, $fn_ex); } main();

Untested.