in reply to Yet another example to get URLs in parallel
package Mojolicious::Command::bulkget; use Mojo::Base 'Mojolicious::Command'; use Mojo::UserAgent; use Mojo::Promise; use Mojo::File 'path'; use Mojo::Util qw(getopt); our $VERSION = '0.03'; my $MAXREQ = 20; has description => 'Perform bulk get requests'; has usage => sub { shift->extract_usage . "\n" }; sub run { my ($self, @args) = @_; getopt \@args, 'v|verbose' => \my $verbose; my ($urlbase, $outdir, $suffixesfile) = @args; die $self->usage . "No URL" if !$urlbase; die $self->usage . "$outdir: $!" if ! -d $outdir; die $self->usage . "$suffixesfile: $!" if ! -f $suffixesfile; my $ua = Mojo::UserAgent->new; # Detect proxy for absolute URLs $urlbase !~ m!^/! ? $ua->proxy->detect : $ua->server->app($self->app +); my $outpath = path($outdir); my @suffixes = _getsuffixes($suffixesfile, $outpath); my @promises = map _makepromise($urlbase, $ua, \@suffixes, $outpath, + $verbose), (1..$MAXREQ); Mojo::Promise->all(@promises)->wait if @promises; } sub _makepromise { my ($urlbase, $ua, $suffixes, $outpath, $verbose) = @_; my $s = shift @$suffixes; return if !defined $s; my $url = $urlbase . $s; warn "getting $url\n" if $verbose; $ua->get_p($url)->then(sub { my ($tx) = @_; _handle_result($outpath, $tx, $s, $verbose); _makepromise($urlbase, $ua, $suffixes, $outpath, $verbose); }); } sub _handle_result { my ($outpath, $tx, $s, $verbose) = @_; if ($tx->res->is_success) { warn "got $s\n" if $verbose; $outpath->child($s)->spurt($tx->res->body); } else { warn "error $s\n" if $verbose; } } sub _getsuffixes { my ($suffixesfile, $outpath) = @_; open my $fh, '<', $suffixesfile or die $!; grep { !-f $outpath->child($_); } map { chomp; $_ } <$fh>; }
|
---|
Replies are listed 'Best First'. | |
---|---|
Re^2: Yet another example to get URLs in parallel
by karlgoethebier (Abbot) on Sep 02, 2024 at 10:30 UTC |