Hi there,
I've written a Perl code to get Foursquare data through its API. The code works fine for about a day or two. Then it starts throwing the following error:
error while setting up ssl connection (SSL connect attempt failed error:1409C041:SSL routines:ssl3_setup_read_buffer:malloc failure)
use strict;
use LWP::UserAgent;
use Data::Dumper qw(Dumper);
use JSON;
use Digest::MD5 qw(md5_hex);
use IO::Handle qw( ); # For flush
my $basedir = 'c:/somedir/';
my $outfile = $basedir.'fs_out_2.txt';
my $last_lat_file = $basedir.'_last_lat.cfg';
my $errorfile = $basedir.'fs_error.txt';
my $client_id = 'xxx';
my $client_secret = 'yyy';
my ($lat,$lng,$decoded,$req,$qurl,$inp,$out,$out2,$s,$i,$j,$content,
+$response,$remaining,$total,$pct,$id,$name,$sw,$ne,$start_lat);
my $user = 'user';
my $pass = 'psw';
my $lat_min = 11.1;
my $lat_max = 12.1;
my $lng_min = 23.4;
my $lng_max = 24.4;
my $geo_inc = .001;
my $limit = 50;
my $v = '20160901';
my $sep = "\t";
my @venues=();
print 'Started: '.(localtime)."\n";
my $url = 'https://api.foursquare.com/v2/venues/search?';
$url .= "client_id=$client_id";
$url .= "&client_secret=$client_secret";
$url .= "&v=$v";
$url .= "&limit=$limit";
$url .= "&intent=browse";
if(-e $last_lat_file){
open($inp,"$last_lat_file") or die "Cannot read $last_lat_file: $
+!\n";
chomp($content=<$inp>);
$start_lat = $content;
close $inp;
}else{
$start_lat = $lat_min;
}
if(-e $outfile){
open($out,">>:encoding(utf8)", $outfile) or die "Cannot write to
+$outfile: $!\n";
}else{
open($out,">:encoding(utf8)", $outfile) || die "Cannot write to $
+outfile: $!\n";
print $out 'id'.$sep.'name'.$sep.'olat'.$sep.'olng'.$sep.'lat'.$s
+ep.'lng'.$sep.'country'.$sep.'city'.$sep.'distance'.$sep.'checkinsCou
+nt'.$sep.'usersCount'.$sep.'cat_shortName'."\n";
};
my $ua = LWP::UserAgent->new(
agent => 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:42.0) Gecko/
+20100101 Firefox/42.0'
);
$ua->proxy('http', 'http://'.$user.':'.$pass.'@my.proxy.server:8080/
+');
$ua->proxy('https', 'connect://'.$user.':'.$pass.'@my.proxy.server:8
+080/');
for ($lat = $start_lat; $lat <= $lat_max; $lat += $geo_inc){
$pct = round(100*($lat-$lat_min)/($lat_max-$lat_min),2);
print "$pct% ".(localtime)."\n";
$out->flush();
open($out2,">$last_lat_file") or die "Cannot write to $last_lat_
+file: $!\n";
print $out2 $lat;
close $out2;
for ($lng = $lng_min; $lng <= $lng_max; $lng += $geo_inc){
$qurl = $url."&sw=$lat,$lng&ne=".($lat+$geo_inc).",".($lng+$g
+eo_inc);
$req = HTTP::Request->new(GET => $qurl);
$req->header(Accept => "text/html,application/xhtml+xml,appli
+cation/xml;q=0.9,*/*;q=0.8",
Accept_Language => "en-US,en;q=0.5",
Connection => "keep-alive"
);
# get response
$response = $ua->request($req);
$remaining = $response->{'_headers'}{'x-ratelimit-remaining'}
+;
if($remaining == 0){
print 'Sleeping for 5 mins: '.(localtime);
if($response->{_rc} ne '200'){
print "; ".$response->{_msg}."\n";
open($out2,">$errorfile") or die "Cannot write to $erro
+rfile: $!\n";
print $out2 Dumper $response;
close $out2;
}else{
print ", srv: ".substr($response->{'_headers'}{'date'},
+5)."\n";
}
$out->flush();
sleep 5*60;
redo;
}
# get content of response
$content = $response->content();
$decoded = decode_json($content);
@venues = @{ $decoded->{'response'}{'venues'}};
foreach $s ( @venues ) {
$name = $s->{'name'};
$name =~ s/\t/ /g;
if(length(trim($s->{'id'})) > 0){
$id = trim($s->{'id'});
}else{
$id = md5_hex($name,$s->{'location'}{'lat'},$s->{'locat
+ion'}{'lng'});
}
print $out $id.$sep;
print $out $name.$sep;
print $out $lat.$sep;
print $out $lng.$sep;
print $out $s->{'location'}{'lat'}.$sep;
print $out $s->{'location'}{'lng'}.$sep;
print $out $s->{'location'}{'country'}.$sep;
print $out $s->{'location'}{'city'}.$sep;
print $out $s->{'location'}{'distance'}.$sep;
print $out $s->{'stats'}{'checkinsCount'}.$sep;
print $out $s->{'stats'}{'usersCount'}.$sep;
print $out $s->{'categories'}[0]{'shortName'}."\n";
}
}
}
close $out;
unlink $last_lat_file;
print 'Finished: '.(localtime)."\n";
sub trim{
my $s = shift @_;
$s =~ s/^\s+|\s+$//g;
return($s);
}
sub round{
my $val = shift;
my $num = shift;
return(int(10**$num*$val+.5)/10**$num);
}
Versions:
Perl: 5.22.1 (Strawberry Perl) Win32 version on 64 bit Win7
LWP: 6.15
LWP::Protocol::connect: 6.09
Any idea what might cause the error?
-
Are you posting in the right place? Check out Where do I post X? to know for sure.
-
Posts may use any of the Perl Monks Approved HTML tags. Currently these include the following:
<code> <a> <b> <big>
<blockquote> <br /> <dd>
<dl> <dt> <em> <font>
<h1> <h2> <h3> <h4>
<h5> <h6> <hr /> <i>
<li> <nbsp> <ol> <p>
<small> <strike> <strong>
<sub> <sup> <table>
<td> <th> <tr> <tt>
<u> <ul>
-
Snippets of code should be wrapped in
<code> tags not
<pre> tags. In fact, <pre>
tags should generally be avoided. If they must
be used, extreme care should be
taken to ensure that their contents do not
have long lines (<70 chars), in order to prevent
horizontal scrolling (and possible janitor
intervention).
-
Want more info? How to link
or How to display code and escape characters
are good places to start.