set 1
-->20030123 041132266-0500 mta2 mta 25955 264 244 Note;MsgTrace(65/26) delivered:user=rolad@localdomain.net:mbox=103999368866444:mss=mxmss05:from=<blahblah@hotmail.com>:msgid=<F137MtkQhhzFSeXOfFd000201dc@hotmail.com>:size=3211:port=5007:fromhost=192.168.4.15.137:localAddr=[192.168.50.100]
-->20030123 041132267-0500 mta2 mta 25955 556 261 Note;MsgTrace(65/26) received from internet:from=<blahblah@hotmail.com>:msgid=<F137MtkQhhzFSeXOfFd000201dc@hotmail.com>:fromhost=[192.168.15.137]:localAddr=192.168.50.100:msgfile=/usr/imail/spool/control/158/20030123091131.TKZC25955.mta2.localdomain.net@hotmail.com-Control:msgsize=3167:time=1:sender=<elizabethhugh80@hotmail.com>:rcpts=<rolad@localdomain.net>

##</code><code>##

-->20030123 043157259-0500 mta2 mta 25955 107 368 Note;MsgTrace(65/26) dropped:user=teacat@localdomain.net:mss=mxmss02:from=<fun@somespamdomain.com>:msgid=<20030123093156.UDXV25955.mta2.localdomain.net@outbound4.la.jackpot.com>:size=4252:port=6003:fromhost=192.168.22.102:localAddr=[192.168.50.100]
-->20030123 043157261-0500 mta2 mta 25955 107 616 Note;MsgTrace(65/26) received from internet:from=<fun@somespamdomain.com>:msgid=<20030123093156.UDXV25955.mta2.localdomain.net@outbound4.la.jackpot.com>:fromhost=[192.168.22.102]:localAddr=192.168.50.100:msgfile=/usr/imail/spool/control/352/20030123093156.UDXV25955.mta2.localdomain.net@outbound4.la.jackpot.com-Control:msgsize=4252:time=1:sender=<fun@somespamdomain.com>:rcpts=<teacat@localdomain.net>

##</code><code>##

-->20030123 010337538-0500 mta2 mta 25955 80 419 Note;MsgTrace(65/26) handled by Error-Handler:from=<aul@localdomain.net>:msgid=<20030123060334.NPQE25955.mta2.localdomain.net@mail>:size=42589:desthost=mailin-01.mx.someremotedomain.com (192.168.138.57):fromhost=192.168.162.207:localAddr=[192.168.50.100]:msgfile=/usr/imail/spool/control/482/20030123060334.NPQE25955.mta2.localdomain.net@mail-Control:msgid=<20030123060334.NPQE25955.mta2.localdomain.net@mail>
-->20030123 010338291-0500 mta2 mta 25955 117 267 Note;MsgTrace(65/26) received from internet:from=<aul@localdomain.net>:msgid=<20030123060334.NPQE25955.mta2.localdomain.net@mail>:fromhost=[192.168.162.207]:localAddr=192.168.50.100:msgfile=/usr/imail/spool/control/482/20030123060334.NPQE25955.mta2.localdomain.net@mail-Control:msgsize=42589:time=4:sender=<Saul@localdomain.net>:rcpts=<blackd57@someremotedomain.com>

##</code><code>##

-->20030123 002524982-0500 mta2 mta 25955 363 762 Note;MsgTrace(65/26) delivered:user=rische@localdomain.net:mbox=103999336928419:mss=mxmss05:from=<ari@someotherdomain.org>:msgid=<NEBBLMPMBNMFIKKABMDGOEEPDLAA.ari@someotherdomain.org>:size=2454:port=5007:fromhost=192.168.10.180:localAddr=[192.168.50.100]
-->20030123 002524984-0500 mta2 mta 25955 205 473 Note;MsgTrace(65/26) received from internet:from=<ari@someotherdomain.org>:msgid=<NEBBLMPMBNMFIKKABMDGOEEPDLAA.ari@someotherdomain.org>:fromhost=[192.168.10.180]:localAddr=192.168.50.100:msgfile=/usr/imail/spool/control/151/20030123052523.MHMQ25955.mta2.localdomain.net@anguilla.alumniconnections.com-Control:msgsize=2419:time=1:sender=<ari@someotherdomain.org>:rcpts=<rische@localdomain.net>

##</code><code>##

-->20030123 014536535-0500 mta2 mta 25955 63 717 Note;MsgTrace(65/26) bounced:user=baner3@localdomain.net:mbox=103999314773141:mss=mxmss01:from=<specials@anotherspamdom.com>:msgid=<20030123064532.OZKU25955.mta2.localdomain.net@mailer72.anotherspamdom.com>:size=10791:port=5007:fromhost=192.168.165.72:localAddr=[192.168.50.100]
-->20030123 014536603-0500 mta2 mta 25955 217 408 Note;MsgTrace(65/26) handled by Error-Handler:mss=mxmss01:from=<specials@anotherspamdom.com>:msgid=<20030123064532.OZKU25955.mta2.localdomain.net@mailer72.anotherspamdom.com>:size=10750:port=5007:fromhost=192.168.165.72:localAddr=[192.168.50.100]:msgfile=/usr/imail/spool/control/448/20030123064532.OZKU25955.mta2.localdomain.net@mailer72.anotherspamdom.com-Control:msgid=<20030123064532.OZKU25955.mta2.localdomain.net@mailer72.anotherspamdom.com>
-->20030123 014545048-0500 mta2 mta 25955 423 314 Note;MsgTrace(65/26) received from internet:from=<specials@anotherspamdom.com>:msgid=<20030123064532.OZKU25955.mta2.localdomain.net@mailer72.anotherspamdom.com>:fromhost=[192.168.165.72]:localAddr=192.168.50.100:msgfile=/usr/imail/spool/control/448/20030123064532.OZKU25955.mta2.localdomain.net@mailer72.anotherspamdom.com-Control:msgsize=10750:time=13:sender=<specials@anotherspamdom.com>:rcpts=<bainter3@localdomain.net>

##</code><code>##

#!/usr/bin/perl

$base  = '/usr/local/stats/data';
$CHARS = '(<|>|\[|\])';

$| = 1;

%states = (
   'bounced'   => 'bounce',
   'deferred'  => 'queued',
   'directory' => 'queued',
   'delivered' => 'del_loc',
   'dropped'   => 'spam',
   'internet'  => 'del_rem',
   'Handler'   => 'err',
   'forwarded' => 'forward',
);

opendir(BASE, "$base") || die "Cant access $base\nReason: $!\n";

foreach $file ( grep(/mta2.*\.log$/, readdir(BASE)) ) {
   chomp($file);

   $real  = "$base/$file";
   $start = time;

   print "Processing: $real\n";
   open(IN, "$real") || die "Cant read $real\nReason: $!\n";

   while ( <IN> ) {
      next if (!/MsgTrace/);
      chomp();
#
# Grab the msgid field, then take the line and stuff it into the id's array after cleaning the
# extra chars from the ID
#
      $id = $1 if (/msgid=([^:]+):/);

      $id =~ s/(^<|>$)//g;

      if (!$id || $id =~ /^(\s+|)$/) {
         $no_id++;
         next;
      }

      push(@{$data{$id}}, $_);
   }
   close(IN);

   print "Finished sorting by msgid in: " . (time - $start) . " seconds\n";

   foreach $id (keys %data) {
      $s_id    = time;
      @data    = @{$data{$id}};
      $r_count = grep(/received/, @data);

      if (!$r_count) {
#
# These are internal messages about queueing and such. We wont keep metrics on this
#
         next;

      } elsif ($r_count >= 2) {
# 
# For some reason we get multiple messages with the same ID.. Not sure how to deal with them
# yet :P
#
         $bad_windows++;
         next;

      } else {
         &process_it(@data);
      }

      @data = ();
      undef(%{$data{$id}});

   } # END foreach id keys %data

   print "Processed: $real in: " . (time - $start) . " seconds\n";

}

print "Totals for all data files.\n";
print <<EOF;
Note: 
   Due to the methods employed to log the mta data, the breakdown by type of mail will not 
   necessarily add up to the total messages sent from an IP. The total is accurate and the
   breakdown may be short data. 

   Also again due to the method data is logged the code could not process < $bad_windows > msg_ids

EOF

foreach $ip ( sort { $by_ips{$a} <=> $by_ips{$b} } keys %by_ips ) {
   next if ($by_ips{$ip} <= 499);

   if ($by_ips{$ip} <= 999) {
      printf("%15s : %s\n", $ip, $by_ips{$ip});

   } else {
      printf("\n%15s : %s\n", $ip, $by_ips{$ip});

      %tmp = %{$by_type{$ip}};

      for (keys %tmp) {
         printf("%15s : %s\n", $_, $tmp{$_});
      }
   }
}

sub Print_Line {
   my($char) = shift;
   if ($char) {
      print "$char" x 80 . "\n";
   } else {
      print '*' x 80 . "\n";
   }
}

sub process_it {
   my($rec_line,$from,$tmp,$count,$line,$type,$tmp_c,$total_c,$unknown, @in,@data,@line);

   @in           = @_;
   ($rec_line)   = grep(/received from internet:/, @in);
   ($from, $tmp) = ( split(/:/, $rec_line) )[3, -1];

   $from =~ s/(fromhost=|$CHARS)//g;
# 
# We convert the @ signs to @ signs, and get back how many times it happened in the string
# in question. This way we know how many people the email went to.
#
   $count = $tmp =~ s/@/@/g;
#
# Grab our data set now, Sometimes we have data sets with a single Error-Handler line,
# other times we have error-handler, and an actual breakdown of what happened to the message
# ie deferred, dropped, bounced, etc.. so if its only one line we leave it alone, else we 
# whack the error-handler lines
# 
   (@data) = grep(!/received from internet:/, @in);
   (@data) = grep(!/Error-Handler/, @data) if ( grep(/Error-Handler/, @data) && $data[1] );

   foreach $line (@data) {
      @line = split(/:/, $line);

      if ($line[0] =~ /( |-)([a-zA-Z]+)$/) {
         $type = $states{$2};
         warn "NO type: $line[0]\n" if ($type =~ /^(\s+|)$/);

         if ($type !~ /del_rem/) {
            $tmp_c++;

         } else {
# 
# this is the same as the above, but we are counting how many people we sent to outside of 
# our mail system
#
            $tmp_c += $line[$#line] =~ s/@/@/g;
         }
      }

      $by_type{$from}{$type} += $tmp_c;
      $total_c               += $tmp_c;
      undef($tmp_c);
   }

   $by_ips{$from}           += $total_c;
}