Beefy Boxes and Bandwidth Generously Provided by pair Networks
Don't ask to ask, just ask
 
PerlMonks  

file download with connection problems

by balushen (Initiate)
on Oct 01, 2010 at 19:24 UTC ( [id://863008]=perlquestion: print w/replies, xml ) Need Help??

balushen has asked for the wisdom of the Perl Monks concerning the following question:

Hi there! I could use some help on a matter, considering I'm Perl rookie. I want to download a file (around 500mb may be) and lets say i've already downloaded 400mb and BAM! my internet connection for some reason resets. When I have my connection back I want to continue downloading the file from 401mb, not downloading the file from the beginning. Thank you in advance Peter!

Replies are listed 'Best First'.
Re: file download with connection problems
by moritz (Cardinal) on Oct 01, 2010 at 19:26 UTC
    wget --continue $url
    Perl 6 - links to (nearly) everything that is Perl 6.
Re: file download with connection problems
by jakeease (Friar) on Oct 02, 2010 at 02:21 UTC

    wget as moritz suggested will definitely work. If you need to do it in a Perl program, you can try LWP.

    use LWP; $have = length($file); $response = $ua->get($URL, 'Range', "bytes=$have-");
      #!/usr/bin/perl -- use strict; use warnings; use LWP; use File::Slurp qw' read_file write_file '; use File::Spec; Main(@ARGV); exit(0); sub Main { my $mech = LWP::UserAgent->new; my $ua = $mech; $ua->add_handler( "request_send", sub { shift->dump; return } ); $ua->add_handler( "response_done", sub { shift->dump; return } ); my $file = File::Spec->rel2abs(__FILE__) . '.temp.html'; #~ print "$file \n"; write_file( $file, ( '#' x ( 5810-9 ) ) ); $ua->resume( 'http://www.cpan.org/', $file ); print "----\n",substr( read_file( $file ), -20 ), "\n\n"; unlink $file; } ## end sub Main sub LWP::UserAgent::resume { my ( $self, $uri, $file, @rest ) = @_; ## TODO: Accept-Ranges ## TODO: If-Range ## TODO: If-Unmodified-Since ## TODO: Last- Modified eval { $uri = $uri->url if ref($uri) eq 'WWW::Mechanize::Link'; $uri = $self->base ? URI->new_abs( $uri, $self->base ) : URI-> +new($uri); }; open my ($fh), '>>', $file or die "Can't open '$file' for append: +$!"; binmode $fh; @rest = ( ':content_cb' => sub { my ( $data, $response, $protocol ) = @_; print $fh $data; return; }, @rest ); my @stat = stat($fh); ## 9 mtime require HTTP::Request::Common; my @suff = $self->_process_colonic_headers( \@rest, 0 ); my $req = HTTP::Request::Common::GET( $uri, @rest ); $req->headers->header( Range => 'bytes=' . $stat[7] . '-' ); @rest = $self->request( $req, @suff, @rest ); close $fh; return @rest; } ## end sub LWP::UserAgent::resume __END__ GET http://www.cpan.org/ Range: bytes=5804- User-Agent: libwww-perl/5.837 (no content) HTTP/1.1 206 Partial Content Connection: close Date: Sat, 02 Oct 2010 01:39:24 GMT Accept-Ranges: bytes ETag: "61c606-16b2-491805f445dc0" Server: Apache/2.2.14 (Unix) Content-Length: 6 Content-Range: bytes 5804-5809/5810 Content-Type: text/html; charset=ISO-8859-1 Last-Modified: Thu, 30 Sep 2010 21:00:47 GMT Client-Date: Sat, 02 Oct 2010 01:40:23 GMT Client-Peer: 207.171.7.177:80 Client-Response-Num: 1 (no content) ---- ########### </html>

Log In?
Username:
Password:

What's my password?
Create A New User
Domain Nodelet?
Node Status?
node history
Node Type: perlquestion [id://863008]
Approved by ww
help
Chatterbox?
and the web crawler heard nothing...

How do I use this?Last hourOther CB clients
Other Users?
Others goofing around in the Monastery: (7)
As of 2024-04-19 13:05 GMT
Sections?
Information?
Find Nodes?
Leftovers?
    Voting Booth?

    No recent polls found