Beefy Boxes and Bandwidth Generously Provided by pair Networks
Keep It Simple, Stupid
 
PerlMonks  

IPTables Log Parser to postgres

by Ryszard (Priest)
on Dec 07, 2002 at 08:11 UTC ( [id://218235]=sourcecode: print w/replies, xml ) Need Help??
Category: Networking Code
Author/Contact Info Ryszard
Description: Dismayed at the platform this uses, I decided to write my own IPTABLES log parser. My own platform is apache, perl and a postgres database, I wasnt about to go and change my database, nor my preferred language just to use it.

Using this module I posted a little while ago the IPTABLES log parser can be modified to use any database backend supported (by DBI), all you have to is work out the differences in the sql.

This module was developed on a RH8.0, perl 5.8.0 and iptables 1.2.6a platform.

The method used is relatively inefficient as it will re-parse the log file each time it is executed and as such uses md5 to determine if the record has been processed before (does a a select for the hash key in the database). This is fine for my personal use, but wont really scale that well..

Personalisations:
•I'm using the the "itp" string to determine what are iptables entries in my syslog.
•Redhat uses /var/log/messages
•Your log format may differ slightly different from mine.. make sure the data you're putting in your DB is sourced from the correct log fields.

package Iptparser;

use strict;
use lib '../lib'; # the path to DBhandler.pm
use DBhandler;
use Digest::MD5 qw /md5_hex/;
use Time::localtime;

{

    sub _execute {
        my $self = shift;
        my %args = @_;

        $self->_fetch_handle if (! $self->{dbo});

        $self->{_dbo}->add_sql( $args{statement} =>
                                $self->_get_statement(statement => $ar
+gs{statement})
                              );
        return $self->{_dbo}->execute(
                                      handle    => 'infomgr',
                                      statement => $args{statement},
                                      bindvar   => $args{bindvar},
                                     );


    }

    sub _get_statement {
        my $self = shift;
        my %args = @_;

        my %sql = (
                   exist => 'select count(*) from stats_iptables where
+ hash = ?',
                   add   => qq/insert into stats_iptables values (next
+val('stats_iptables_stat_id_seq'),
                               ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?
+, ?, ?, ?, ?, ?, ?, ?, ?, ?)/,
                  );

        return $sql{$args{statement}};
    }

    # Grab a db_handle on demand
    sub _fetch_handle {
        my $self = shift;
        $self->{_dbo} = DBhandler->new(
                                       handle =>'infomgr',
                                       sid    =><sid>,
                                       user   =><user>,
                                       pwd    =><passwd>,
                                      );

    }

    sub _check {
        my $self = shift;
        my %args = @_;
        return $self->_execute(handle    => 'infomgr',
                               statement => 'exist',
                               bindvar   => [$args{bindvar}],);
    }

    sub _add {
        my $self = shift;
        my %args = @_;

        return $self->_execute(handle    => 'infomgr',
                               statement => 'add',
                               bindvar   => $args{bindvar},);
    }
}

sub new {
    my $pkg = shift;


    my $self = bless { }, $pkg;
    return $self;
}

sub gather_data {
    my $self = shift;

    my $logfile = '/var/log/messages';
    local*LOGFILE;
    open (LOGFILE, $logfile) || die "Cannot open $logfile for processi
+ng: $!";

    while (<LOGFILE>) {
 
        next unless /ipt\:/;
        chomp;

        my $digest = md5_hex($_);
        my @log = split(/ /, $_);

        # Go grab the "header" information
        my ($mon, $day, $time, $host, $chain) = @log[0,2,3,4,7];

        # Go grab the rest of the log information
        my %hash;
        foreach (@log) {
             next unless ($_ =~ /=/);
             my @tmp = split(/=/, $_);
             $hash{$tmp[0]}=$tmp[1];
        }
        my $year = localtime->year() + 1900;

        my %month = (Jan=>1,Feb=>2,Mar=>3,Apr=>4,May=>5,Jun=>6,
                     Jul=>7,Aug=>8,Sep=>9,Oct=>10,Nov=>11,Dec=>12);

        my $check = $self->_check(bindvar => $digest);
        if ($check->[0] < 1) {
            my $retval = $self->_add(bindvar => ["$year-$month{$mon}-$
+day $time",$digest, $hash{IN}, $host, $chain,
                                     $hash{OUT},$hash{MAC},$hash{SRC},
+$hash{DST},$hash{LEN},$hash{TOS},
                                     $hash{PREC},$hash{TTL},$hash{ID},
+$hash{PROTO},$hash{SPT},$hash{DPT},
                                     $hash{SEQ},$hash{ACK},$hash{WINDO
+W},$hash{RES},$hash{RST},$hash{URGP}]);

        };

    }

}
1

The SQL

DROP TABLE stats_iptables;
DROP SEQUENCE stats_iptables_stat_id_seq;

CREATE TABLE stats_iptables (
                        stat_id         serial primary key,
                        date            timestamp, 
                        hash            varchar,
                        inbound         varchar,
                        host            varchar,
                        chain           varchar,
                        outbound        varchar,
                        mac             varchar,
                        src             varchar,
                        dst             varchar,
                        len             integer,
                        tos             varchar,
                        prec            varchar,
                        ttl             integer,
                        id              integer,
                        proto           varchar,
                        spt             integer,
                        dpt             integer,
                        seq             varchar,
                        ack             varchar,
                        window          varchar,
                        res             varchar,
                        rst             varchar,
                        urgp            varchar
                    );

create index hash_idx on stats_iptables (hash);
create index date_idx on stats_iptables (date);

grant select on stats_iptables to apache;
grant insert on stats_iptables to apache;
grant update on stats_iptables to apache;
grant delete on stats_iptables to apache;
grant update on stats_iptables_stat_id_seq to apache;

Log In?
Username:
Password:

What's my password?
Create A New User
Domain Nodelet?
Node Status?
node history
Node Type: sourcecode [id://218235]
help
Chatterbox?
and the web crawler heard nothing...

How do I use this?Last hourOther CB clients
Other Users?
Others avoiding work at the Monastery: (6)
As of 2024-04-24 16:47 GMT
Sections?
Information?
Find Nodes?
Leftovers?
    Voting Booth?

    No recent polls found