my $logtime = timelocal(${sec}, ${min}, ${hour},
${mday}, ${mon}, ${current_year} - ${new_year});
####
sub getLogsSCP {
my @hosts = @_;
dprint "Getting logs for the last ${cutoff} hours";
dprint " from " . @hosts . " hosts\n";
# Build command to list log files
# Using find allows us to scan only files that are new enough
my $ssh_command = "find";
$ssh_command .= " @messages_files";
$ssh_command .= " -mtime -" . (int(${cutoff} / 24) + 1);
my $nHosts = 1;
foreach (@hosts) {
chomp;
my $host = $_;
$current_host = ${host};
$last{${host}} = "";
my $c = 0;
while () {
chomp;
my $file = $_;
my $tmpfile = "/tmp/logmon.${host}.$hnow{year}$hnow{mon}$hnow{mday}.$ENV{LOGNAME}";
# Copy log file
dprint localtime() . " Copying ${file} to ${tmpfile}\n";
my $scp = Net::SCP->new({"host" => ${host}});
$scp->get(${file}, ${tmpfile});
# Process log file
my $size = (stat("${tmpfile}"))[7];
if (! defined(${size})) {
dprint localtime() . " Failed to copy ${file} from ${host}\n";
addFakeMessage(${host}, "FAILED TO COPY ${file} FROM ${host}");
$total_count++;
next;
}
dprint localtime() . " Parsing ${tmpfile} (${size} bytes)\n";
open(FD, "${tmpfile}") or next;
unlink "${tmpfile}";
$c += processFile(\*FD, ${tmpfile}, ${host});
close(FD);
}
push @emptyhosts, ${host} if ($c == 0);
$total_count += $c;
close(*READER);
close(*WRITER);
close(*ERR);
}
dprint "\n";
}
# Process a log file (or stream)
sub processFile() {
my $FD = shift; # Caller must open and close file descriptor
my $file = shift;
my $host = shift;
my $c = 0;
my $line = 1;
while () {
# Log file is too big
if ($line++ > ${max_messages}) {
dprint "\n" . localtime() . " Too many messages in ${file}";
addFakeMessage(${host}, "STOPPED PROCESSING ${file} AFTER ${max_messages} MESSAAGES (please rotate)");
$total_count++;
last;
}
dprint "${line}..." if (($line % 10000) == 0);
my $retval += addMessage($_);
if (${retval} < 0) {
$c = 0;
} else {
$c += ${retval};
}
}
dprint "\n" if (${line} > 10000);
return ${c} ;
}
sub addMessage($) {
my $msg = shift;
chomp ${msg};
# Skip blank lines
if (${msg} =~ /^$/) { $reason = "blank"; return 0; }
# Parse message
my ($month, $mday, $tm, $host, @message) = split(/ ?/, ${msg});
my ($hour, $min, $sec) = split(/:/, ${tm}) if (defined(${tm}));
# Skip line if we don't know the hostname
if (! defined(${host}) || ${host} eq "") { $reason = "hostname" ; return 0; }
# Skip lines that don't look like syslog messages
unless (${tm} =~ /[0-2]\d:[0-5]\d/) { $reason = "tm = ${tm}"; return 0; }
# Skip lines that are too old
my $mon = $month_to_mon{${month}};
my $logtime = timelocal(${sec}, ${min}, ${hour},
${mday}, ${mon}, ${current_year} - ${new_year}); <----------------- This line is producing the error.
if ((${mon} == 0) && (${mday} == 1) && (${new_year} == 1)) {
# Special case for Jan 1 (only!)
$logtime = timelocal(${sec}, ${min}, ${hour},
${mday}, ${mon}, ${current_year});
}
$logtime -= ((int($offsets{${host}}) / 100) * 100)
if (defined($offsets{${host}}));
if (${logtime} < ${then}) { $reason = "too old"; return 0; }
# Reset host if line is from the future; we may have messages from one
# year ago. Note that this may make the global counts wrong.
if (${logtime} > (time + 3600)) { # Allow 1 hour slack time
$last{${host}} = "";
$counts{${host}} = 0;
delete($messages{${host}});
undef($times{${host}}{oldest});
undef($times{${host}}{newest});
$reason = "future";
return -1;
}
my %month_to_mon = # Convert month names to numbers (Jan == 0)
( "Jan" => 0, "Feb" => 1, "Mar" => 2, "Apr" => 3,
"May" => 4, "Jun" => 5, "Jul" => 6, "Aug" => 7,
"Sep" => 8, "Oct" => 9, "Nov" => 10, "Dec" => 11,
"JAN" => 0, "FEB" => 1, "MAR" => 2, "APR" => 3,
"MAY" => 4, "JUN" => 5, "JUL" => 6, "AUG" => 7,
"SEP" => 8, "OCT" => 9, "NOV" => 10, "DEC" => 11);
my @mon_to_month = # Convert month numbers to names (0 == Jan)
qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec);
####
--------------------host1----------------
Feb 2 22:58 host1 SAPBJD_03[17692]: [ID 503851 user.error] Unable to open trace file sapstartsrv.log. (Error 11 Resource temporarily unavailable) [ntservsserver.cpp 3231] Feb 2 22:58 host1 SAPBJD_02[19005]: [ID 503851 user.error] Unable to open trace file sapstartsrv.log. (Error 11 Resource temporarily unavailable) [ntservsserver.cpp 3231] Feb 2 23:00 burton SAPBWD_00[22453]: [ID 365072 user.error] Unable to open trace file sapstartsrv.log. (Error 11 Resource temporarily unavailable) [ntservsserver.cpp 3405]
--------------------host2----------------
--------------------host3----------------
Permission denied, please try again.
Permission denied, please try again.
Permission denied (publickey,password,keyboard-interactive).
--------------------host4----------------
--------------------host5----------------
ssh: connect to host host5 port 22: Connection timed out
--------------------host6----------------
Feb 2 04:08 host6 automountd[29387]: [ID 834250 daemon.error]