In the spirit of "more than one way", Mojolicious::Command::bulkget, in its entirety: 
package Mojolicious::Command::bulkget;
use Mojo::Base 'Mojolicious::Command';
use Mojo::UserAgent;
use Mojo::Promise;
use Mojo::File 'path';
use Mojo::Util qw(getopt);
 
our $VERSION = '0.03';
 
my $MAXREQ = 20;
 
has description => 'Perform bulk get requests';
has usage => sub { shift->extract_usage . "\n" };
 
sub run {
  my ($self, @args) = @_;
  getopt \@args,
    'v|verbose'              => \my $verbose;
  my ($urlbase, $outdir, $suffixesfile) = @args;
  die $self->usage . "No URL" if !$urlbase;
  die $self->usage . "$outdir: $!" if ! -d $outdir;
  die $self->usage . "$suffixesfile: $!" if ! -f $suffixesfile;
  my $ua = Mojo::UserAgent->new;
  # Detect proxy for absolute URLs
  $urlbase !~ m!^/! ? $ua->proxy->detect : $ua->server->app($self->app
+);
  my $outpath = path($outdir);
  my @suffixes = _getsuffixes($suffixesfile, $outpath);
  my @promises = map _makepromise($urlbase, $ua, \@suffixes, $outpath,
+ $verbose), (1..$MAXREQ);
  Mojo::Promise->all(@promises)->wait if @promises;
}
 
sub _makepromise {
  my ($urlbase, $ua, $suffixes, $outpath, $verbose) = @_;
  my $s = shift @$suffixes;
  return if !defined $s;
  my $url = $urlbase . $s;
  warn "getting $url\n" if $verbose;
  $ua->get_p($url)->then(sub {
    my ($tx) = @_;
    _handle_result($outpath, $tx, $s, $verbose);
    _makepromise($urlbase, $ua, $suffixes, $outpath, $verbose);
  });
}
 
sub _handle_result {
  my ($outpath, $tx, $s, $verbose) = @_;
  if ($tx->res->is_success) {
    warn "got $s\n" if $verbose;
    $outpath->child($s)->spurt($tx->res->body);
  } else {
    warn "error $s\n" if $verbose;
  }
}
 
sub _getsuffixes {
  my ($suffixesfile, $outpath) = @_;
  open my $fh, '<', $suffixesfile or die $!;
  grep { !-f $outpath->child($_); } map { chomp; $_ } <$fh>;
}

[download]

In reply to Re: Yet another example to get URLs in parallel by etj
in thread Yet another example to get URLs in parallel by karlgoethebier

Title:
Use:  <p> text here (a paragraph) </p>
and:  <code> code here </code>
to format your post, it's "PerlMonks-approved HTML":


  • Posts are HTML formatted. Put <p> </p> tags around your paragraphs. Put <code> </code> tags around your code and data!
  • Titles consisting of a single word are discouraged, and in most cases are disallowed outright.
  • Read Where should I post X? if you're not absolutely sure you're posting in the right place.
  • Please read these before you post! —
  • Posts may use any of the Perl Monks Approved HTML tags:
    a, abbr, b, big, blockquote, br, caption, center, col, colgroup, dd, del, details, div, dl, dt, em, font, h1, h2, h3, h4, h5, h6, hr, i, ins, li, ol, p, pre, readmore, small, span, spoiler, strike, strong, sub, summary, sup, table, tbody, td, tfoot, th, thead, tr, tt, u, ul, wbr
  • You may need to use entities for some characters, as follows. (Exception: Within code tags, you can put the characters literally.)
            For:     Use:
    & &amp;
    < &lt;
    > &gt;
    [ &#91;
    ] &#93;
  • Link using PerlMonks shortcuts! What shortcuts can I use for linking?
  • See Writeup Formatting Tips and other pages linked from there for more info.