#!/usr/bin/perl use strict; use Digest::MD5 qw(md5); my (@lines, %line_md5); while (<>) { unless (/^\s*$/) { my $digest = md5($_); unless ( exists $line_md5{$digest} ) { $line_md5{$digest} = 1; push @lines, $_; } } else { push @lines, $_; } } print @lines; #### cat file | sort | uniq