perl -ne 'print if!$h{$_}++' file1 file2 ... > newfile



##</code><code>##

    perl -pe '$_=$,if$,{$_}++' file1 file2 ... > newfile