package logparse;

sub new {
    return bless { LEX => { '\w+' => 'TOK_ID',
                            '^[:punct:]+' => 'TOK_PUNCT',
                            # and so on for all character classes you identify
                 }};
}
 
sub lex { 
    my $self = shift;
    my $fh = $self -> { FH };
    $self -> { BUFFER } ||= <$fh> or goto EOF;
    PAT: while ( my ($pat, $tok) = each %{ $self -> { LEX }} ) {
        $/^($pat)(.*)$/ or next PAT;
        $self -> { BUFFER } = $2;
        $self -> { LEXVAL } = $1;
        return $tok;
    }

    $self -> { LEXVAL } = substr( $self -> { BUFFER }, 0, 1 );
    $self -> { BUFFER } =~ s/^.//;
    warn "unhandled content at $fh line $.\n";
    return '';

EOF: $self -> { LEXVAL } = '';
     return 'TOK_EOF';
}