nysus has asked for the wisdom of the Perl Monks concerning the following question:

This post is a follow up to a question I asked yesterday. I've created an object called RemoteFile that I use to perform various operations on a file located on another server. Here is the entire class:

package RemoteFile 0.000001; use Carp; use Moose; use Modern::Perl; use File::Basename; use File::Slurper qw(read_text write_text); use Params::Validate; with 'MyOpenSSH', 'MyLogger2', 'DownloadDir'; use namespace::autoclean; has 'path' => (is => 'ro', isa => 'Str', required => 1, lazy => 0, writer => '_set_path', default => '' ); has 'gid' => (is => 'rw', isa => 'Int', required => 0, lazy => 1, defa +ult => '' ); has 'uid' => (is => 'rw', isa => 'Int', required => 0, lazy => 1, defa +ult => '' ); has 'perms' => (is => 'rw', isa => 'Value', required => 0, lazy => 1, +default => '' ); has 'filename' => (is => 'rw', isa => 'Str', required => 0, lazy => 1, + default => '' ); has 'localfile' => (is => 'rw', isa => 'Str', required => 0, lazy => 1 +, default => '' ); sub BUILD { my $self = shift; my $path = $self->path; $self->logt('building remotefile'); # make sure something exists at path given my $exists = $self->grab(_one_liner("-e('$path')")); $self->logf("$path does not exist at " . $self->get_host) if !$exist +s; # get absolute path of file if it is a symlink my $is_symlink = undef; while ($is_symlink = $self->grab(_one_liner("-l('$path')"))) { my $dir = dirname($path); my $cmd = _one_liner("Cwd::abs_path(readlink('$path'))", 'Cwd', $d +ir); my $abs_path = $self->grab($cmd); $self->_set_path($abs_path); $path = $abs_path; } # make sure we are dealing with a file my $is_file = $self->_file_exists($path); $self->logf("$path is not a file at " . $self->get_host) if !$is_fil +e; # fetch data about the file my $uid = $self->grab(_one_liner("(stat '$path')[4]")); my $gid = $self->grab(_one_liner("(stat '$path')[5]")); my $mode = $self->grab(_one_liner("(stat '$path')[2]")); my $perms = sprintf ("%04o", $mode & 07777); my ($filename) = fileparse($path); # set some object attributes $self->gid($gid); $self->uid($uid); $self->perms($perms); $self->filename($filename); $self->localfile($self->dl_dir . $self->filename); # make sure we have a fresh copy of file and download to local machi +ne unlink ($self->dl_dir . $self->filename); my %rsync_options = (rsync_path => 'sudo rsync', quiet => 1); $self->rsync_get(\%rsync_options, $self->path, $self->dl_dir); } sub append_file { my $self = shift; $self->logi('Appending local file'); validate_pos( @_, 1 ); my $text_to_append = shift; my $local_file = $self->localfile; open(my $fh, '>>', $local_file) or $self->logf("could not open $loca +l_file to append"); print $fh $text_to_append; close $fh; } sub sandr { my $self = shift; my $search = shift; my $replace = shift; $self->logi('Search $'); my $text = read_text($self->localfile, 'UTF-8', 1); $text =~ s/$search/$replace/gim; write_text($self->localfile, $text, 'UTF-8', 1); } sub read_and_delete { my $self = shift; my $content = read_text($self->localfile, 'UTF-8', 1); unlink ($self->localfile); return $content; } sub upload_file { my $self = shift; $self->logi('Uploading file to server'); my $local_file = $self->localfile; my $path = shift || $self->path; my $staging_file = "/home/me/tmp/tmp_file"; my $is_file = $self->_file_exists($staging_file); $self->exec("sudo rm $staging_file") if $is_file; $self->scp_put($local_file, $staging_file) or $self->logf("Could not + cp $local_file to server."); $self->exec('sudo chmod ' . oct($self->perms) . " $staging_file"); $self->exec("sudo chown root:root $staging_file"); $self->exec("sudo cp $staging_file $path"); $self->exec("sudo rm $staging_file"); unlink($self->localfile); $self->logi('File uploaded successfully.'); }

The RemoteFile object uses MyOpenSSH as a role which is a wrapper for Net::OpenSSH. Here is the code:

package MyOpenSSH 0.000001; use Carp; use Data::Dumper; use Moose::Role; use Modern::Perl; use Net::OpenSSH; use Params::Validate; with 'MyLogger2'; has 'ssh' => (is => 'rw', isa => 'Net::OpenSSH', required => 1, lazy +=> 0, handles => qr/.*/, ); has 'connection' => (is => 'ro', isa => 'Str', required => 1, lazy => +0); around BUILDARGS => sub { my $orig = shift; my $class = shift; my %args = ref $_[0] ? %{$_[0]} : @_; croak 'a host must be supplied for ssh: ssh => (\'<user>@<host>\', % +opts)' if !%args; my ($host) = $args{ssh}; if (ref $host eq 'Net::OpenSSH') { my $user = $host->get_user; my $host = $host->get_host; $args{connection} = $user . '@' . $host; return $class->$orig( %args ); } delete $args{ssh}; my %ssh_args = map { $_ => $args{$_ } } grep { exists $args{$_ } } q +w( user port password passphrase key_path gateway proxy_command batch +_mode ctl_dir ssh_cmd scp_cmd rsync_cmd remote_shell timeout kill_ssh +_on_timeout strict_mode async connect master_opts default_ssh_opts fo +rward_agent forward_X11 default_stdin_fh default_stdout_fh default_st +derr_fh default_stdin_file default_stdout_file default_stderr_file ma +ster_stdout_fh master_stderr_fh master_stdout_discard master_stderr_d +iscard expand_vars vars external_master default_encoding default_stre +am_encoding default_argument_encoding password_prompt login_handler m +aster_setpgrp); my $ssh = Net::OpenSSH->new($host, %ssh_args); $ssh->error and croak "could not connect to host: $ssh->error"; return $class->$orig( ssh => $ssh, %args, connection => $ssh->get_us +er . '@' . $ssh->get_host); }; # check to make sure the command can't do anything destructive on the +server sub _check_command { my $self = shift; my @cmds = @_; shift @cmds if ref $cmds[0]; my @restricted_cmds = qw (rm chmod chown chgrp); my $restricted_cmds = join '|', map { "($_)" } @restricted_cmds; my $restricted_cmd_regex = qr/\b$restricted_cmds\s+/; # scan args to see if they have a restricted command my $is_restricted = grep { $_ =~ $restricted_cmd_regex } @cmds; return if !$is_restricted; # put command in one string my $cmd = ''; if (@cmds > 1) { my $cmd = join ' ', @cmds; } else { $cmd = shift @cmds; } # chop off sudo commands $cmd =~ s/^\bsu(do)*\s+//gi; my @multiple_cmds = split /&&/, $cmd; @multiple_cmds = map { split /;/ } @multiple_cmds; my @safe_dirs = qw (/var/www/ /Users/me/tmp/ /home/me/tmp/ /home/sit +es/wp_sites/ /home/dir/preview); my @safe_dir_regexes = map { s/\//\\\//g; qr/$_.+/; } @safe_dirs; foreach $cmd (@multiple_cmds) { if ($cmd =~ $restricted_cmd_regex) { if (! grep { $cmd =~ /$_/} @safe_dir_regexes ) { $self->logf("Not authorized to perform that operation in that +directory: $cmd"); } if ($cmd =~ /\s(\/|\b)[^\s]*\/[^\s]*(\/|\b)\s+(\/|\b).*\b/) { $self->logf('Cannot run restricted commands on more than file +or directory at a time.'); } if ($cmd =~ /(\s\*)|(\s\*$)|(\s\/\s)|(\s\/$)|(\s\/\s)|(\s\/$)|(\ +s\.\.)|(\s\.\.$)/) { $self->logf("Not authorized to perform that operation in that +directory: $cmd"); } } } } sub _file_exists { my $self = shift; my $file = shift; return $self->grab(_one_liner("-f('$file')")); } sub _dir_exists { my $self = shift; my $dir = shift; return $self->grab(_one_liner("-d('$dir')")); } sub _one_liner { my $cmd = shift; my $mod = shift; my $cd = shift; my $one_liner = 'perl '; $one_liner .= $mod ? "-M$mod -e " : '-e'; $one_liner = $cd ? "cd $cd && " . $one_liner : $one_liner; return qq#$one_liner "print ($cmd)"#; } # wrapper for system method sub exec { my $self = shift; $self->_check_command(@_); return $self->ssh->system(@_) if !$self->ssh->error; croak ('ssh command failed: ' . $self->ssh->error . ", $!"); } # wrapper for capture method sub grab { my $self = shift; $self->_check_command(@_); return $self->ssh->capture(@_) if !$self->ssh->error; croak ('ssh command failed: ' . $self->ssh->error); } sub disconnect { my $self = shift; $self->ssh->disconnect; }

Now, my problem comes in after I download and read a couple hundred files from the server, the program craps out and I can't create anymore connections. It happens despite trying to disconnect from the SSH connection with this function:

sub get_file { validate_pos(@_, 1, 1); my $self = shift; my $file_path = shift; if (!$self->_file_exists($file_path)) { $self->logw("File $file_path does not exist"); $self->logt('checking file'); return ''; } my $file = RemoteFile->new({ path => $file_path, ssh => $self->connection } ); my $content = $file->read_and_delete; # disconnect/destroy the file object's ssh connection $file->disconnect; return $content; }

I cannot figure out how to kill off the SSH connections which apparently use up file handlers that I no longer need. Any help is appreciated. Perhaps it has something to do with using the rsync_get command?

$PM = "Perl Monk's";
$MCF = "Most Clueless Friar Abbot Bishop Pontiff Deacon Curate";
$nysus = $PM . ' ' . $MCF;
Click here if you love Perl Monks

Replies are listed 'Best First'.
Re: Need help finding cause of leaking file handlers
by Corion (Patriarch) on Mar 28, 2017 at 16:09 UTC

    You have posted lots of code.

    What steps have you taken to try to isolate the problem?

    You still load lots of modules that you haven't shown us:

    with 'MyOpenSSH', 'MyLogger2', 'DownloadDir';

    Which of these modules is unneccessary? Can you remove some of them and makes that the problem persist or go away?

    Maybe your logger opens a new filehandle for each instance.

    Please reduce your code to something of about 20 or 30 lines at most that still reproduces the problem.

      Corion strikes again! I disabled the MyLogger2 module and that turned out to be the culprit. It opens a file handle to append to a log file. I didn't even stop to think about that. Thanks for the help!

      $PM = "Perl Monk's";
      $MCF = "Most Clueless Friar Abbot Bishop Pontiff Deacon Curate";
      $nysus = $PM . ' ' . $MCF;
      Click here if you love Perl Monks