in reply to Parsing Apache logs with Regex
From my toolbox: I threw this together a while back to parse log files and load them into sqlite. Its not pretty but it works. Feel free to use what you need.
#!/usr/bin/perl # # Parses logfiles & loads to sqlite db use strict; use warnings; use DBI; #### CONFIG # FILES my $logfile = 'access_log'; my $dbfile = 'acclog.sdb'; # TABLES my $newlog = 'logentries'; my $oldlog = 'oldlog'; # IF needed - creates new input table & renames old one # REMEMBER to edit the table names above create(); #### END CONFIG my @names = qw(ip id user datime req status bytes referer agent); my @cols = qw( ip id user date time zone method bytes status url proto type datime req referer agent); my $colstr = join( ',', @cols ); my @places; for (@cols){push @places, '?'} my $places = join ',', @places; my $dbh = DBI->connect("DBI:SQLite:$dbfile") or die 'connect fail'; my $sql = qq(INSERT INTO `$newlog` ($colstr) values ( $places ) ); my $sth = $dbh->prepare($sql); open my $FH, "$logfile" or die "cannot open file: $logfile\n"; while (my $line = <$FH>){ my %dat; my @fields = $line =~ m/("[^\"]*"|\[.*\]|[^\s]+)/g; $fields[3] =~ s/[\[\]]//g; ($dat{date}, $dat{time}) = split /:/, $fields[3], 2; ($dat{time}, $dat{zone}) = split / /, $dat{time}, 2; $fields[4] =~ s/"//g; #" ($dat{method}, $dat{url}, $dat{proto}) = split / /, $fields[4]; if ($dat{url} =~ /\/$/){ $dat{type} = 'dir'; }else{ ($dat{type}) = $dat{url} =~ /(\.\w+)$/g; } $dat{type} = 'file' unless $dat{type}; $fields[7] =~ s/"//g; #" $fields[8] =~ s/"//g; #" for (0..$#names){ $dat{ $names[$_] } = $fields[$_]; } my @insert; for (@cols){ push @insert, $dat{$_}; } $sth->execute(@insert); } close $FH; $sth->finish(); $dbh->disconnect(); sub create{ my $dbh = DBI->connect("DBI:SQLite:$dbfile") or die 'connect faile +d'; my $zql = qq(ALTER TABLE $newlog RENAME TO $oldlog); $dbh->do($zql) or die 'rename failed'; my @cols = qw( ip id user date time zone method bytes status url proto typ +e datime req referer agent); my $colstr = join( ',', @cols ); my $sql = qq(CREATE TABLE $newlog (seq INTEGER PRIMARY KEY AUTOINC +REMENT, $colstr) ); $dbh->do($sql) or die 'create failed'; $dbh->disconnect(); }
|
|---|
| Replies are listed 'Best First'. | |
|---|---|
|
Re^2: Parsing Apache logs with Regex
by Anonymous Monk on Jan 24, 2009 at 18:46 UTC |