in reply to 400 error

Hey guys, thanks for replying and giving me information. Please look at the code, the output says 400 url is missing. The complete code. 
#!/usr/local/bin/perl
use LWP::Simple;
use HTML::Parse;
use HTML::Element;
use URI::URL;
use HTTP::Request;
use HTTP::Tiny;
use vars qw($opt_h $opt_i $opt_a);
use Getopt::Std; 
 

#system "cls";
#print "\n\n";

getopts('hia:');
my $all = !($opt_i || $opt_a);    # all=1 when no option is set

if ($opt_h || $#ARGV==-1) {  print_help( ); # print help text when -h 
+or no args
  exit(0);
}

sub print_help {
  print <<"HELP";
usage: $0 [-hia] [URL]
 
 -h help
 -i find img references only
 -a find hyper link references only
 
Example:  $0 -a https://www.url_here.com
 
HELP
}

$html        = get $ARGV[0]; # always use [0]
$parsed_html = HTML::Parse::parse_html($html);

while ($html = shift @ARGV) {
my ($code, $type, $data)  = get_html($html, $opt_i, $opt_a);
if (not_good($code, $type)) { next; }
if ($opt_i || $all) { print_images($data, $html); 
exit(0);
}
if ($opt_a || $all) { print_hyperlinks($data, $html); 
exit(0);
}
}
sub get_html( ) {
  my($html, $want_image, $want_link) = @_;
 
# Create a User Agent object
  my $ua = new LWP::UserAgent;
  $ua->agent("Mozilla/61.0.0 (Windows; U; Windows 8.1; de; rv:1.9.2.3)
+ Gecko/20100401 Firefox/61.0.0 ");
 
 
# Ask the User Agent object to request a URL.
# Results go into the response object (HTTP::Reponse).
#  use HTTP::Request::Common;
#  my $request = GET $html;
#  my $request = GET $ARGV[0];

my $url = $html;

print "$response->{status} $response->{reason}\n";
print $response->{content} if length $response->{content};
    
my $request = new HTTP::Request('GET', $ARGV[0]);
my $response = HTTP::Tiny->new->get($url);

my $response = $ua->request($request);
if ($response->is_success) {
     print $response->content;
} else {
     print $response->error_as_HTML;
}
print "$response->{status} $response->{reason}\n";
print $response->{content} if length $response->{content};
}
  
sub not_good {

  my ($code, $type) = @_;
 
  if ($code != RC_OK) {
    warn("$html had response code of $code");
    return 1;
  }
 
  if ($type !~ m@text/html@) {
    warn("$html is not HTML.");
    return 1;
  }
  return 0;
}  

sub print_images {
 
  my ($data, $model) = @_;
 
  my $parsed_html=HTML::Parse::parse_html($data);
  for (@{ $parsed_html->extract_links(qw (body img)) }) {
    my ($link) = @$_;
    my ($absolute_link) = globalize_url($link, $model);
    print "$absolute_link\n";
  }
  $parsed_html->delete( ); # manually do garbage collection
}
 
sub print_hyperlinks {
 
  my ($data, $model) = @_;
 
  my $parsed_html=HTML::Parse::parse_html($data);
  for (@{ $parsed_html->extract_links(qw (a)) }) {
    my ($link) = @$_;
    my ($absolute_link) = globalize_url($link, $model);
    print "$absolute_link\n";
  }
  $parsed_html->delete( ); # manually do garbage collection
}

sub globalize_url( ) {
 
  my ($partial, $model) = @_;
  my $url = new URI::URL($partial, $model);
  my $globalized = $url->as_string;
  return $globalized;
}

[download]

Replies are listed 'Best First'.
Re^2: 400 error
by 1nickt (Canon) on Dec 02, 2018 at 00:39 UTC

    Hi, I'll be glad to look at it after it's properly indented and formatted. (But you might find your error once you do that...) I can see already that there's a lot of what we call krufty kruft in there, i.e. leftovers from things you tried that you're now not using. For example you are loading HTTP::Tiny and LWP::Simple. You should clean that up too. Finally, you should shorten the program for posting here so it only has the code for the thing that breaks. The rest is just obscuring the issue, for us and for you. See SSCCE.

    Thanks!


    The way forward always starts with a minimal test.
[reply]
[d/l]
[select]
      Thank you for guiding, this is the code which I think creating all sorts of issues. Please let me know if you think the code is perfect. 
       
  my $ua = new LWP::UserAgent;
  $ua->agent("myagent");
 
 
my $url = $html;
my $request = new HTTP::Request('GET', $ARGV[0]);
my $response = HTTP::Tiny->new->get($url);


my $response = $ua->request($request);
if ($response->is_success) {
     print $response->content;
} else {
     print $response->error_as_HTML;
}

      [download]
[reply]
[d/l]
        my $response = HTTP::Tiny->new->get($url);


my $response = $ua->request($request);

        [download]

        Unfortunately, the code isn't perfect. You've declared and assigned to $response twice on subsequent lines and therefore the effect of the former line is nil. If you had used warnings this would have been clear.

        This isn't an SSCCE because you are referring to $ARGV[0] without indication of what it might be and you have not used either of the modules. Here is an SSCCE for contrast.

        #!/usr/bin/env perl 
use strict;
use warnings;
use LWP::UserAgent;

my $ua       = LWP::UserAgent->new (agent => 'myagent');
my $url      = 'https://www.perlmonks.org/';
my $request  = HTTP::Request->new ('GET', $url);
my $response = $ua->request ($request);

print $response->status_line . "\n";

        [download]
[reply]
[d/l]
[select]

        Hi, as hippo pointed out, this is not an SSCCE, and you are declaring $response twice. Additionally:

        • To be clear, you do *not* need to create a request object manually if you use HTTP::Tiny (nor LWP::UserAgent for that matter)
        • While we don't know how you have stored a URL in $html, it's likely not a good name for a variable that does not contain HTML.
        • You probably don't want to print the error as HTML if you are not returning output to a browser
        You need to stop throwing in snippets from various sources and settle on one tool, and take the time to learn it. That involves spending time with the documentation. The example I originally showed was copied almost verbatim from the HTTP::Tiny documentation. I suggest that you start with the SYNOPSIS of the module that you choose, and once you have that running in your environment, you can make changes to it, one line at a time, so you can see immediately if you made a mistake.

        This is an SSCCE that should be copied into a file (eg 'foo.pl') and run with $ perl foo.pl http://example.com 

        use strict;
use warnings;
use HTTP::Tiny;

my $url = shift;
my $response = HTTP::Tiny->new->get($url);

if ( $response->{success} ) {
     print "OK: $response->{content}\n";
} else {
     print "Failed: $response->{status} $response->{reason}\n";
}

__END__

        [download]
        ... or this ... 
        use strict;
use warnings;
use LWP::UserAgent;

my $ua = new LWP::UserAgent;
# possibly add advanced user agent options here

my $url = shift;
# possibly construct a request object here

my $response = $ua->get($url); # if not using a request object

if ($response->is_success) {
    print $response->decoded_content, "\n";
} else {
    die $response->status_line;
}

__END__

        [download]

        Hope this helps! There are no shortcuts to learning, not if you want to understand.


        The way forward always starts with a minimal test.
[reply]
[d/l]
[select]

In Section Seekers of Perl Wisdom