use strict;
use warnings;

use WWW::Mechanize;
use WWW::RobotRules;
use LWP::Simple;

my $SITE  = 'http://www.unlocalhost.com';
my $rules = WWW::RobotRules->new('bot/1.0');

my $robot_url  = "$SITE/robots.txt";
my $robot_data = LWP::Simple::get($robot_url);
$rules->parse($robot_url, $robot_data) if $robot_data;

for ('disallow.txt', 'allow.txt') {
   my $url = "$SITE/$_";
   if($rules->allowed($url)) {
      my $mech = WWW::Mechanize->new;
      $mech->get($url);
      print "$url:\n", $mech->content;
   }
   else {
      print "$url:\ndenied\n";
   }
}