use strict;
use warnings;

my $ff = 'robots.txt';
my $fh;
my @lines;

# Read the entire file and
# store lines in an array
open $fh, "<", $ff or die "Sorry, can't open file - $ff\n";
{
  local $/;
  @lines = split("\n", <$fh>);
}
close $fh;

# Get rid of duplicate lines
@lines = sort(@lines);
my $L;
my $prev = '';
foreach $L (@lines)
{
  print($L . "\n") if ($prev ne $L);
  $prev = $L;
}