#!/usr/bin/perl -w
# Logregate
# Artem Litvinovich, 2001-08-02
# an aggregate log analyzer
# aims to provide output from various log sources in a time syncronize
+d manner
# logregate utility by Artem Litvinovich
use strict;
use Config::General;
use Getopt::Std;
use Time::Local;
use POSIX;
my %logtable = ();
my @sections = ();
my ($interval) = ("");
&main();
exit 1;
sub main {
my (%months_3ltr,%months_full) = ();
@months_3ltr{ qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec)
+} =
@months_full{ qw(January February March April May June July August
+ September October November December) } = (0..11);
my %reserved = (
'default_timestamp' => [ qw( ^($M=\w+)\b($D=\d+)\b($h=\d+
+):($m=\d+):($s=\d+)\b($data=.*)$ )], # timestamp works on syslog file
+s
'default_interval' => ["1m"],
'default_maxlines' => ["-1"],
'default_nostamp' => ["log"],
'default_fromtime' => ["00,00,00,12,02,1982"],
'default_totime' => ["59,59,23,30,11,2020"]
);
# get command line options
my %options = (); getopt('c:',\%options);
my $conFile = exists($options{c}) ? $options{c} : "./logregate.con
+f";
die "Unable to find config file ($conFile).\nUsage: logregate [-c
+config_file]\n"
unless(-f $conFile);
# load config file
my $conf = new Config::General($conFile);
my %config = $conf->getall;
# get grouping interval
$interval = exists($config{default_interval}) ? interval2sec($conf
+ig{default_interval}) : interval2sec($reserved{default_interval});
foreach my $section (keys %config) {
next if(exists $reserved{$section});
# check with path, then without, store if found, next if doubt
my ($logfile) = exists($config{$section}{path}) ? (-f $config{$sec
+tion}{path}."/".$section ? ($config{$section}{path}."/".$section) : n
+ext) :
(-f $section ? ($section) : next);
# conf must have a way to get the timestamp, use default if regex
+omitted
my ($stampRegex,@elements) = exists($config{$section}{timestamp})
+? stamp2regex($config{$section}{timestamp}) :
(exists($config{default_timestamp}) ? stamp2regex($config{defa
+ult_timestamp}) : $reserved{default_timestamp});
# get maxlines read from logfile
my $maxlines = exists($config{$section}{maxlines}) ? $config{$sect
+ion}{maxlines} :
(exists($config{default_maxlines}) ? $config{default_maxlines}
+ : $reserved{default_maxlines});
# get nostamp action
my $nostamp = exists($config{$section}{nostamp}) ? $config{$sectio
+n}{nostamp} :
(exists($config{default_nostamp}) ? $config{default_nostamp} :
+ $reserved{default_nostamp});
# get from time limit
my $fromTime = exists($config{$section}{fromtime}) ? $config{$sect
+ion}{fromtime} :
(exists($config{default_fromtime}) ? $config{default_fromtime}
+ : $reserved{default_fromtime});
$fromTime = timelocal(split(",",$fromTime));
# get to time limit
my $toTime = exists($config{$section}{totime}) ? $config{$section}
+{totime} :
(exists($config{default_totime}) ? $config{default_totime} : $
+reserved{default_totime});
$toTime = timelocal(split(",",$toTime));
open LOGDATA, $logfile;
my %logline = ('Y'=>'2001');
my $epochSeconds = 0;
# main log-reading loop
while(my $line = <LOGDATA>) {
chomp($line);
$maxlines--; # if maxlines was set >0, loop until it is 0.
if( $line =~ m/$stampRegex/ ) { # extract timestamp
# record each timestamp element (+data) into logline hash
for(my $ecount=0;$ecount<scalar(@elements);$ecount++) {
$logline{$elements[$ecount]} = eval("\$".($ecount+1));
}
# record entire line if data not specified
$logline{data} = $line unless($logline{data});
# when getting month, try to get numerical value
$logline{M} = exists($months_3ltr{$logline{M}}) ? $months_3lt
+r{$logline{M}} :
(exists($months_full{$logline{M}}) ? $months_full{$logline
+{M}} : ($logline{M} =~ /^\d*$/ ? $logline{M} : next));
# convert timstamp into a single number (hash key)
$epochSeconds = timelocal($logline{s},$logline{m},$logline{h},
+$logline{D},$logline{M},$logline{Y});
} else { # did not find timstamp with regex from conf, examine
+ alternatives
for ($nostamp) {
/ignore/ && do {
last;
};
/log/ && do {
$logline{data} = "[NO TIME-STAMP] ".$line;
};
/incby\s+(\d+)/ && do {
$epochSeconds+=$1;
};
/decby\s+(\d+)/ && do {
$epochSeconds-=$1;
};
}
}
# add data to overall table
push @{$logtable{$epochSeconds}{$section}}, $logline{data} if(
+$epochSeconds>=$fromTime && $epochSeconds<=$toTime);
last unless($maxlines);
}
close LOGDATA;
push @sections, $section;
}
&displayHTML();
}
sub display {
my $mark = -1;
foreach my $entrytime (sort keys %logtable) {
if($entrytime-$mark > $interval) {
print "time: ".localtime($entrytime)."\n";
$mark = $entrytime;
}
foreach my $section (keys %{$logtable{$entrytime}}) {
print "section: $section\n";
foreach (@{$logtable{$entrytime}{$section}}) {
print "entry: $_\n";
}
}
}
}
sub displayHTML {
my $mark = -1;
my $colWidth = "300";
my $minColorDiv = "00";
my $maxColorDiv = "5";
my $baseColor = "9933";
my $colorStep = "51";
my $colorOpr = "+";
my $colorMod = $minColorDiv;
print "<HTML><BODY>\n";
print "Interval is $interval seconds.<br>\n";
# display deviation spectrum
print "<TABLE><TR><td colspan=".($maxColorDiv+1).">Interval separa
+tion color spectrum in seconds:</td></TR>\n";
print "<TR>\n";
for ($minColorDiv..$maxColorDiv) {
print "<td bgcolor=".($baseColor.($_?dec2hex($colorStep*$_):"00"))
+.">".
sqr($_)*$interval.
"</td>\n";
}
print "</TR><br>\n<TR>\n";
for ($minColorDiv..$maxColorDiv) {
print "<td bgcolor=".($baseColor.($_?dec2hex($colorStep*$_):"00"))
+.">".sqr($maxColorDiv+$minColorDiv-$_)*$interval."</td>\n";
}
print "</TR></TABLE><br>\n";
# main agregate log table
print "<TABLE border=1 width=".$colWidth*scalar(@sections).">\n";
foreach my $entrytime (sort keys %logtable) {
if($entrytime-$mark > $interval) { # if interval reached, reprint
+heading
my $intDiff = POSIX::floor(sqrt(($entrytime-$mark)/$interval)-
+1);
if($colorMod >= $minColorDiv+$colorStep*$maxColorDiv) {
$colorOpr = "-";
} elsif ($colorMod <= $colorStep*$minColorDiv) {
$colorOpr = "+";
}
$colorMod += ($colorOpr.$colorStep * $intDiff);
if($colorMod>$minColorDiv+$colorStep*$maxColorDiv) {
$colorMod = $minColorDiv+$colorStep*$maxColorDiv;
} elsif($colorMod<$colorStep*$minColorDiv) {
$colorMod = $colorStep*$minColorDiv;
}
my $color = $baseColor . ($colorMod ? dec2hex($colorMod) : "00
+");
print "<TR><td colspan=".scalar(@sections)." bgcolor=#$color><
+b>time: ".localtime($entrytime)."</b></td></TR>\n";
$mark = $entrytime;
print "<TR>";
foreach (sort @sections) {
print "<td><b>$_</b></td>\n";
}
print "</TR>";
}
print "<TR>";
foreach my $section (sort @sections) {
print "<td width=$colWidth>";
foreach (exists($logtable{$entrytime}{$section}) ? @{$logtable
+{$entrytime}{$section}} : ()) {
print localtime($entrytime)." $_<br>\n";
}
print "</td>\n";
}
print "</TR>";
}
print "</TABLE>\n";
print "</BODY></HTML>\n";
}
sub stamp2regex {
# s - sec, m - min, h - hour, D - day, W - week, M - month, Y - ye
+ar
# Ex:
# "^($M=\w+)\b($D=\d+)\b($h=\d+):($m=\d+):($s=\d+)\b($data=.*)$" f
+or syslog
# Aug 2 16:57:59 artemz1 su(pam_unix)[18581]: session closed for
+user root
my ($str) = @_;
my @orderedElements = ();
# drop encapsulating quotes
$str =~ s/^"(.*)"$/$1/;
# compile the time_stamp elements in order
while($str =~ m/\(\$(.+?)=.+?\)/g) {
push @orderedElements, $1;
}
# convert into a real regex
$str =~ s/\(\$.+?=(.+?)\)/\($1\)/g;
return ($str,@orderedElements);
}
sub dec2hex($) {
my($dec) = @_;
return sprintf("%lx ", $dec );
}
sub sqr($) {
my($num) = @_;
return $num*$num;
}
sub interval2sec($) {
# convert interval to seconds
# s - sec, m - min, h - hour, D - day, W - week, M - month, Y - ye
+ar
my ($interval) = @_;
$interval =~ s/^(\d+)(\w)$/$1/;
for ($2) {
/s/ and last;
$interval *= 60;
/m/ and last;
$interval *= 60;
/h/ and last;
$interval *= 24;
/D/ and last;
$interval *= 7;
/W/ and last;
$interval *= 4;
/M/ and last;
$interval *= 12;
/Y/ and last;
$interval = $1; # wierd denomination, assuming seconds.
}
return $interval;
}
-------------
Sample conf file:
# list of logs
# time interval default
# time interval for each
# s - sec, m - min, h - hour, D - day, M - month, Y - year
#
default_timestamp "^($M=\w+)\s+($D=\d+)\s+($h=\d+):($m=\d+):($s=\d+)\s
++($data=.*)$"
default_interval 5m
default_nostamp log
default_fromtime 00,00,10,6,7,2001
default_totime 00,00,15,6,7,2001
<./u/messages>
</./u/messages>
<./u/maillog>
</./u/maillog>
<./u/cron>
timestamp "^.*?\(($M=\d+).($D=\d+).($h=\d+):($m=\d+):($s=\d+).($data=.
+*)$"
</./u/cron>
<./uniserv/messages>
</./uniserv/messages>
<./uniserv/maillog>
</./uniserv/maillog>
<./uniserv/cron>
timestamp "^.*?\(($M=\d+).($D=\d+).($h=\d+):($m=\d+):($s=\d+).($data=.
+*)$"
</./uniserv/cron>
<./uniserv/secure>
</./uniserv/secure>
In reply to logregate
by ArtemZ1
Posts are HTML formatted. Put <p> </p> tags around your paragraphs. Put <code> </code> tags around your code and data!
Titles consisting of a single word are discouraged, and in most cases are disallowed outright.
Read Where should I post X? if you're not absolutely sure you're posting in the right place.
Please read these before you post! —
Posts may use any of the Perl Monks Approved HTML tags:
- a, abbr, b, big, blockquote, br, caption, center, col, colgroup, dd, del, details, div, dl, dt, em, font, h1, h2, h3, h4, h5, h6, hr, i, ins, li, ol, p, pre, readmore, small, span, spoiler, strike, strong, sub, summary, sup, table, tbody, td, tfoot, th, thead, tr, tt, u, ul, wbr
You may need to use entities for some characters, as follows. (Exception: Within code tags, you can put the characters literally.)
| |
For: |
|
Use: |
| & | | & |
| < | | < |
| > | | > |
| [ | | [ |
| ] | | ] |
Link using PerlMonks shortcuts! What shortcuts can I use for linking?
See Writeup Formatting Tips and other pages linked from there for more info.