sub tokenize_msg_w_strings {
    my ($msg) = @_;

    # define consitituent characters.
    my $con = 'A-Za-z0-9\'\$!,.-';

    # separate tokens with % for later splittage
    $msg =~ s/[^$con]+/%/g;          # replace any non-cons with %
    $msg =~ s/%[,.]/%/g;             # delete [,.] in the front of a token
    $msg =~ s/[,.]%/%/g;             # delete [,.] in the back of a token
    $msg =~ s/^[,.]//g;              # delete [,.] at the start of a line
    $msg =~ s/[,.]$//g;              # delete [,.] at the end of a line
    $msg =~ s/(?<=\D)[,.](?=\D)/%/g; # delete [,.] not surrounded by digits

    my %words = map {$_=>1} split /%/, $msg;

    return keys %words;
}