#!/usr/bin/perl

use strict;

open( IN, "<:encoding(iso8859-1)", $ARGV[0] ) or die "couldn't read $ARGV[0]: $!";

binmode STDOUT, ":encoding(iso8859-1)";

while (<IN>) {
    my @words = ( /\b(\w+)\b/g );
    print join "\n", @words;
    print "\n";
}
# run it like this: tokenizer latin1.txt > latin1.tkns