#!/usr/bin/perl  

#WGET-STYLE DOWNLOADER VIA LWP::UserAgent;

use strict;
use warnings;
use LWP::UserAgent;
use HTTP::Cookies;
use Encode qw(encode decode);

my $user_agent  = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36';

my $http_header = 'Accept: application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5; Accept-Charset: UTF-8, iso-8859-1; Accept-Language: en-US,en;q=0.5; Accept-Encoding: gzip,deflate; Connection: keep-alive';

# [snip] skip to pertinent subroutine . . .

sub fetch { if ($DEBUG) { print "SUB fetch: $_ \n" };
my $url       = shift @_;
my $agency = $user_agent || 'Mozilla/5.0 (Windows NT 6.1)';

   my ($method, $uri, $data) = @ARGV;
   $method = 'GET' unless $method;
   $uri = $url unless $uri;
   $data = '' unless $data; 

   my $browser = new LWP::UserAgent();
   $browser->cookie_jar( {} );
   $browser->requests_redirectable(['GET', 'HEAD', 'POST']);
   $browser->max_redirect(5);
   $browser->agent($agency);   
   $browser->get($url);    

   my $request = HTTP::Request->new();
           $request->method($method);
          $request->uri($uri);
           if (uc($method) eq 'POST') {
               if ($http_header) { 
                   $request->header($http_header);
               } else {
                  $request->header('Content-Type' => 'application/x-www-form-urlencoded');
              }
              $request->content($data);
           }

   my $jar = HTTP::Cookies->new();
   $jar->load('./HTTP-Cookies.jar');        #LOCAL FILE, WRITABLE
   $jar->add_cookie_header($request);

   my $response = $browser->request($request);
   $request = $response->request();
   $jar->extract_cookies($response);
   $jar->save('HTTP-Cookies.jar');

    $response = decode("utf8", $response->as_string());
    unless ($keep_response_header) {
    
        #TO REMOVE THE HTTP HEADERS (NOTE: _NOT_ THE HTML HEADERS)
        #~~~I COULDN'T THINK OF A WORKABLE ONE-LINER FOR THIS~~~
        #---------
        my @lump = split(/\n/, $response);
        my $hdr = 1;
        my $line = '';
        while (($hdr) && (@lump)) {
            $line = shift @lump;
            if ($line =~ m/^\s*$/) {$hdr=0};
        }
        $response = join "\n", @lump;
        #---------
    }
   
   return $response;

} #END SUB fetch


##</code><code>##

#LWP-Cookies-1.0
Set-Cookie3: 1P_JAR=2022-04-24-16; path="/"; domain=.google.com; path_spec; secure; expires="2022-05-24 16:49:33Z"; version=0
Set-Cookie3: AEC=AakniGM0siZKGKmyFVveOarPvbRyMhhgILvlobJdmPIlHSZDzBcH9ydhdZs; path="/"; domain=.google.com; path_spec; secure; expires="2022-10-21 16:49:33Z"; HttpOnly; SameSite=lax; version=0
Set-Cookie3: NID="511=b4k5SZAGi5bJDr41ZOmk-PAN1cFp0SiGD39_9e4AyeoCoHqycr9_QS13X_oMwyA055BRm46An2txQ9XYUI0QZK8zU2j5NP_BGmVBHyrDggE_NzYqDVzk5NU1Q2PzPEvenKLIVkPXQVbJTM664h7byByPmnioIKx3vvYpjxr_314"; path="/"; domain=.google.com; path_spec; expires="2022-10-24 16:49:33Z"; HttpOnly; version=0