#!/usr/bin/perl -w

use LWP::Simple;
use HTML::SimpleLinkExtor;
use Data::Dumper;
use LWP::RobotUA;
use HTTP::Response;

open(LINKS,">>/home/baelnorn/urls.txt") || die "$!";

while(<LINKS>) {
	print"hello";
	chomp $_;
	my $ua = LWP::RobotUA->new("theusefulbot", "akurtis3 at yahoo.com");
	$ua->delay(10/60);
	my $content= $ua->get($_);
	my $extor = HTML::SimpleLinkExtor->new();
	$extor->parse($content);

	my @links=$extor->a;
	print "start";
	foreach $links (@links) {
		if($links=~m/^\// and $_=~m/\/$/) {
			substr($links, 0, 1) = undef;
			print "1";
			my $address="$_ $links";
			print LINKS "$address\n";
		} else  {
				if($links=~m/^http:\/\/|^www./) {
						print LINKS "$links\n";
				}

				if($links != ~m/^\// and $_=~m/\/$/) {
					my $address="$_ $links";
					print LINKS  "$address\n";
				}

				if($links != ~m/^\// and $_ != ~m/\/$/) {
					my $address="$_ \ $links";
					print LINKS  "$address\n";
				}

			}
	}
	print $content;
}
close(LINKS);