sub tokenize_msg_w_lists {
    my ($msg) = @_;

    # define consitituent characters.
    my $con = 'A-Za-z0-9\'\$!,.-';

    # first pass. split tokens on any non-consitituent characters.
    my @words = map {s/[,.]$//; $_} split /[^$con]+/, $msg;

    # second pass. split on , or . that aren't surrounded by digits.
    my %words;
    foreach (@words) { $words{$_}++ foreach split /(?<=\D)[,.](?=\D)/, $_; }

    return keys %words;
}

##</code><code>##

sub tokenize_msg_w_strings {
    my ($msg) = @_;

    # define consitituent characters.
    my $con = 'A-Za-z0-9\'\$!,.-';

    # separate tokens with % for later splittage
    $msg =~ s/[^$con]+/%/g;          # replace any non-cons with %
    $msg =~ s/%[,.]/%/g;             # delete [,.] in the front of a token
    $msg =~ s/[,.]%/%/g;             # delete [,.] in the back of a token
    $msg =~ s/^[,.]//g;              # delete [,.] at the start of a line
    $msg =~ s/[,.]$//g;              # delete [,.] at the end of a line
    $msg =~ s/(?<=\D)[,.](?=\D)/%/g; # delete [,.] not surrounded by digits

    my %words = map {$_=>1} split /%/, $msg;

    return keys %words;
}