use strict;

my $file = 'data.txt';
my $thiscount;
my $fullcount;
my $max = 10; # change this to, say, 10000
my %lines;

open(INPUT, $file);
while(<INPUT>){
  chomp;
  if(exists $lines{$_}){
    print "duplicate line (on read):$_\n";
  }
  else{
    $lines{$_} = 1;
  }
  $thiscount ++;
  $fullcount ++;
  if($thiscount >= $max){
    my $checkcount=0;
    open(CHECK, $file);
    while(<CHECK>){
      $checkcount ++;
      if($checkcount > $fullcount){
        chomp;
        if(exists $lines{$_}){
          print "duplicate line (on check):$_\n";
        }
      }
    }
    undef %lines;
    $thiscount = 0;
  }
}