#! perl -sw
use strict;
use 5.010;
use Digest::MD5 qw[ md5 ];
use String::CRC32;
use Devel::Size qw[ total_size ];

open IN, '<', 'randStr-1M(64-254).dat' or die $!;

my %asc;
chomp, ++$asc{ $_ } while <IN>;
printf "%07d Ascii keys: %.f\n", scalar keys( %asc ), total_size( \%asc );
undef %asc;

seek IN, 0, 0;
my %md5;
chomp, undef( $md5{ md5( $_ ) } ) while <IN>;

printf "%07d binary MD5 keys: %.f\n", scalar keys( %md5 ),  total_size( \%md5 );
undef %md5;

seek IN, 0, 0;
my %crc;
chomp, undef( $crc{ pack 'VV', crc32( $_ ), crc32( scalar reverse $_ ) } ) while <IN>;

printf "%07d binary CRC keys: %.f\n", scalar keys( %crc ),  total_size( \%crc );


__END__
c:\test>bigHash.pl
1000000 Ascii keys: 53879053
1000000 binary MD5 keys: 35766510
1000000 binary CRC keys: 34756892