First, the version that uses list ops:

sub tokenize_msg_w_lists {
    my ($msg) = @_;

    # define consitituent characters.
    my $con = 'A-Za-z0-9\'\$!,.-';

    # first pass. split tokens on any non-consitituent characters.
    my @words = map {s/[,.]$//; $_} split /[^$con]+/, $msg;

    # second pass. split on , or . that aren't surrounded by digits.
    my %words;
    foreach (@words) { $words{$_}++ foreach split /(?<=\D)[,.](?=\D)/,
+ $_; }

    return keys %words;
}
[download]

And here is the version that uses string ops:

sub tokenize_msg_w_strings {
    my ($msg) = @_;

    # define consitituent characters.
    my $con = 'A-Za-z0-9\'\$!,.-';

    # separate tokens with % for later splittage
    $msg =~ s/[^$con]+/%/g;          # replace any non-cons with %
    $msg =~ s/%[,.]/%/g;             # delete [,.] in the front of a t
+oken
    $msg =~ s/[,.]%/%/g;             # delete [,.] in the back of a to
+ken
    $msg =~ s/^[,.]//g;              # delete [,.] at the start of a l
+ine
    $msg =~ s/[,.]$//g;              # delete [,.] at the end of a lin
+e
    $msg =~ s/(?<=\D)[,.](?=\D)/%/g; # delete [,.] not surrounded by d
+igits

    my %words = map {$_=>1} split /%/, $msg;

    return keys %words;
}
[download]