#!/usr/bin/perl

use strict;
use warnings;

use LWP::UserAgent;
use HTML::LinkExtor;
use URI::URL;
use DBI();
my $url = <>;    # for instance

#my $depth = 0;
my @link = ();
my $ua   = LWP::UserAgent->new;    # Set up a callback that collect links
my @a    = ();

sub callback {
  my( $tag, %attr ) = @_;
  return if $tag ne 'a';
  push( @a, values %attr );
} # Make the parser.Unfortunately, we don't know the base yet (it might be
#diffent from $url)

my $p = HTML::LinkExtor->new( \&callback );
my $res = $ua->request( HTTP::Request->new( GET => $url ),
                        sub { $p->parse( $_[0] ) } )
  ;    # Expand all image URLs to absolute ones
my $base = $res->base;
@a = map { $_ = url( $_, $base )->abs; } @a;    # Print them out
print join( "\n", @a ), "\n";

my $dbh = DBI->connect( "DBI:mysql:database=gatxp;host=\"\"", "", "" );

#$dbh->do(" CREATE TABLE newlinks( md5 INTEGER(100) not null "
#         ."primary key, webpage VARCHAR(80) not null) ");
$dbh->do(" INSERT INTO newlinks VALUES( 'MD5', '0', '$base', '1' ) ");
foreach $a (@a) {
  $dbh->do(" INSERT INTO newlinks VALUES( '', '1', '$a', '0' ) ");
}
my $sth = $dbh->prepare('SELECT  *  FROM newlinks')
  or die " Couldn't prepare statement : " . $dbh->errstr;
$sth->execute();
while( my $ref = $sth->fetchrow_hashref() ) {
  my $link = $ref->{'webpage'};
  foreach $link (@link) {
    my $usa = LWP::UserAgent->new;
    $p = HTML::LinkExtor->new( \&callback );
    my $res = $usa->request( HTTP::Request->new( GET => $link ),
                             sub { $p->parse( $_[0] ) } );
    $base = $res->base;
    @link = map { $_ = url( $_, $base )->abs; } @link;    # Print them out
    print "$$link \n ";
  }
}
$sth->finish();
$dbh->disconnect();

##</code><code>##

s**lil*;  $*=join'',sort split q**;  s;.*;grr; &&s+(.(.)).+$2$1+; $; =
qq-$_-;s,.*,ahc,;$,.=chop for split q,,,reverse;print for($,,$;,$*,$/)