in reply to Re: Requesting webpages which use cookies and session ids. (rev)
in thread Requesting webpages which use cookies and session ids. (rev)

Thanks dws - but its still trowing up the same problem as outlined above

What might I be doing wrong - is there any reason why the cookies wouldn't be working. ANd if not, is it Env. Vars that I need to look at.
Thanks.

I've edited the code anyway and here it is -
#!/usr/bin/perl -w use strict; use URI; use LWP::UserAgent; use HTTP::Request; use HTTP::Headers; use HTTP::Response; use HTTP::Cookies; use HTTP::Request::Common qw(GET POST); my $url_home = "http://www.bt.co.uk/directory-enquiries/dq_home.jsp"; my $ua = new LWP::UserAgent(); $ua->cookie_jar(HTTP::Cookies->new(file => "lwpcookies.txt",autosave = +> 1)); # Get a session ID first my $req = GET $url_home; my $res1 = $ua->request($req); die $res1->as_string() . "\n" if $res1->is_error(); die "Can't find a session ID!\n" unless ($res1->as_string() =~ /BV_Ses +sionID=([^&]+)\&/); my $sessID = $1; die "Can't find an engine ID!\n" unless ($res1->as_string() =~ /BV_Eng +ineID=([^&]+)\&/); my $engID = $1; #die "Can't find a Cookie ID!\n" unless ($res1->as_string() =~ /BV_IDS +=([^;]+)\;/); $cookie_jar->extract_cookies($res1); # Save the cookie jar's state print "Cookies: ",$cookie_jar->as_string(),"\n"; $cookie_jar->save($cookie_file); ###################### Start Searching # too lazy for urlencode... $sessID =~ s/\@/%40/g; my $request = POST $url_search, [ QRY => 'res', BV_SessionID => $sessID, BV_EngineID => $engID, new_search => 'true', NAM => 'Griffin', GIV => '', LOC => '', STR => '', PCD => 'BT', limit => '50', CallingPage => 'Homepage', ]; $cookie_jar->load; $cookie_jar->add_cookie_header($request); my $res2 = $ua->request($request); ###################### How many BT** on this page # deleted ###################### Reveal Second Page Results die $res2->as_string() . "\n" if $res2->is_error(); die "Can't find a session ID!\n" unless ($res2->as_string() =~ /BV_SessionID=([^&]+)\&/); $sessID = $1; die "Can't find an engine ID!\n" unless ($res2->as_string() =~ /BV_EngineID=([^&]+)\&/); $engID = $1; print STDERR "Got session ID $sessID\n"; print STDERR "Got engine ID $engID\n"; # too lazy for urlencode... $sessID =~ s/\@/%40/g; $request = POST $url_search, [ QRY => 'res', NAM => 'Griffin', lci => '0', PCD => 'BT', start_id => '50', CallingPage => 'Homepage', ]; my $res3 = $ua->request($request); ################### Print 3 pages to http://baz.perlmonk.org/save.html open (LOG,">save.html"); my $fileOut = $res1->as_string().$res2->as_string().$res3->as_string() +; print LOG "$fileOut";
  • Comment on Re: Re: Requesting webpages which use cookies and session ids. (rev)
  • Download Code

Replies are listed 'Best First'.
Re: Re: Re: Requesting webpages which use cookies and session ids. (rev)
by Jenda (Abbot) on Aug 04, 2002 at 22:22 UTC

    You say you are too lazy for urlencode, but do it partialy anyway. I believe you should not. The HTTP::Request::Common::POST() should take care of that. Please try to comment out the

    # too lazy for urlencode... $sessID =~ s/\@/%40/g;

      Jenda

      Sorry Jenda, I missed your post....I commented out that conversion line and now I'm getting

      To use this service you will need either an Internet Explorer (IE) browser or Netscape 4.7 and above.

      for the first search page now as well as for subsequent pages - at least theres some consistency there. :)