my $ua = Spider::LWP->new($depth,$path,$max_sockets,$ignore,$exclude);
$ua->duplicates(0);#don't ignore duplicates here as this is done in the subclass a gazillion times more efficiently
$ua->cookie_jar({});#where else would you store cookies?
$ua->redirect(1);#follow redirects.  HACKED BASE LIBRARY to make this work with the subclass.
$ua->in_order(1);#do the urls in order, as we randomize their entry into the queue.
$ua->remember_failures(0);#don't remember failures here as the lib stores the entire object.  This is done in the subclass
$ua->max_hosts($max_sockets);#max open requests at any given moment
$ua->max_req(1);#max requests per host
$ua->nonblock(1);#don't block on LWP::UserAgent socket reads

##</code><code>##

 $ua->wait(300); # block until we are all finished or until everything has stopped for 5 minutes