#!/usr/bin/perl

use strict;
use warnings;

use LWP::UserAgent;
use HTML::LinkExtor;
use URI::URL;

my $site = shift @ARGV;
my $domain = shift @ARGV;
my $firsturl = "http://$site";
my $ua = LWP::UserAgent->new;
my @links = ();
my @newlinks = ();
my @visited = ();
my $newlink = "";
my $link = "";
push (@links, $firsturl);

# Set up a callback that collects links
sub callback {
   my($tag, %attr) = @_;
   return if $tag ne 'a';  # we only look closer at <a ...>
   push(@newlinks, values %attr);
}     

# Make the parser
my $p = HTML::LinkExtor->new(\&callback);

# The main loop
MAIN: foreach my $url (@links) {
	# Skip if we have been here before        
        foreach my $inside (@visited) {
		#print "skipping $url\n" if $url eq $inside;
                next MAIN if $url eq $inside;
        }

	# Request document and parse it as it arrives
	print "visiting $url\n";
	my $res = $ua->request(HTTP::Request->new(GET => $url),
                      sub {$p->parse($_[0])});

	# Remember that we have visited this url
	push (@visited, $url);

	# Expand all URLs to absolute ones
	my $base = $res->base;
	@newlinks = map { $_ = url($_, $base)->abs; } @newlinks;

	# Reduce the links to only ones in our domain
	foreach $newlink (@newlinks) {               
        	if ($newlink =~ /$domain/) {
                        push (@links, $newlink);
		}
	}   
}