use strict; my $sql = <<'--SQL--'; # this is a comment $a = "simple string"; $b = "string and comment"; # comment $c = "string with # thing"; # comment with "string" $d = "very hard" # dasdas . "to do"; --SQL-- use HOP::Lexer 'string_lexer'; my $lexer = string_lexer( $sql, [ STRING => qr/"[^"]+"/ ], [ COMMENT => qr/#[^\n]*/ ], [ SPACE => qr/\s+/, sub {} ], ); # (continues like the others...) #### use strict; my $sql = <<'--SQL--'; # this is a comment $a = "simple string"; $b = "string and comment"; # comment $c = "string with # thing"; # comment with "string" $d = "very hard" # dasdas . "to do"; --SQL-- use HOP::Lexer 'string_lexer'; my $lexer = string_lexer( $sql, [ STRING => qr/"[^"]+"/ ], [ COMMENT => qr/#/ ], [ NEWLINE => qr/\n/ ], [ SPACE => qr/\s+/, sub {} ], ); # parse my @out; my $comment = 0; while (defined (my $token = $lexer->())) { if (ref $token) { my ($label, $value) = @$token; $comment = 1 if $label eq 'COMMENT'; $comment = 0 if $label eq 'NEWLINE'; next if $label eq 'NEWLINE'; } push @out, $token if not $comment; } # Data::Dump the output (elaborated in order to produce compact results) use Data::Dumper; $Data::Dumper::Indent = 0; $Data::Dumper::Terse = 1; ($\, $,) = ("\n", ",\n"); print map { Dumper $_ } @out; #### '$a', '=', ['STRING','"simple string"'], ';', '$b', '=', ['STRING','"string and comment"'], ';', '$c', '=', ['STRING','"string with # thing"'], ';', '$d', '=', ['STRING','"very hard"'], '.', ['STRING','"to do"'], ';'