my %lookup; open IN, '<', ... open OUT, '>', ... while( ) { my( $file, $line, $rule ) = m[...(...)...(...)...(...)] or warn("Bad data: '$_' line: $.\n") and next; if( exists $lookup{ join $;, $file, $line, $rule } ) { ## Duplicate, discard next; } else { ## New, record. $lookup{ join $;, $file, $line, $rule } = 1; print OUT; ## output $_ } } close IN; close OUT;