#!/usr/bin/perl -w

use LWP::Simple;
use HTML::SimpleLinkExtor;
use Data::Dumper;
use LWP::RobotUA;
use HTTP::Response;

open(LINKS,">>/home/baelnorn/urls.txt");

while(<LINKS>) {
chomp $_;
my $ua = LWP::RobotUA->new("theusefulbot", "akurtis3 at yahoo.com");
$ua->delay(.10/60);
my $content= $ua->get("$_");
my $extor = HTML::SimpleLinkExtor->new();
$extor->parse($content);

my @links=$extor->a;

foreach $links (@links) {
if($links=~m/^\// and $_=~m/\/$/) {
substr($links, 0, 1);
my $address="$_ $links";
print LINKS "$address\n";
} else  {
	if($links=~m/^http:\/\/|^www./) {
	print LINKS "$links\n";
}

if($links != ~m/^\// and $_=~m/\/$/) {
	my $address="$_ $links";
	print LINKS  "$address\n";
}

if($links != ~m/^\// and $_ != ~m/\/$/) {
	my $address="$_ \ $links";
	print LINKS  "$address\n";
}

}
}
print $content;
}