#!/usr/bin/perl

use strict;
use warnings;
use HTML::Entities;

my $config = {
  minw =>  3,
  maxw => 20,
};

my $stop = get_stop();
my $punc = get_punc();

my $text = q|
  cat's dogs 
  O’Reilly 
  ad-hoc 
  “broken”
  hyphen- 
  the and
|;

my $words_all = {};

# contrived loop to show usage
for my $t ($text){
  get_word($t);
}

print "$_\n" for keys %{$words_all};  

sub get_word{
  
  my ($text, $file_key) = @_;
  my ($min, $max) = ($config->{minw}, $config->{maxw});
  
  for ($text){
    s/’|'/'/g;
    s/(&#?\w+;)/exists $punc->{$1}?' ':$1/eg;
  }
  
  decode_entities($text);
  $text =~ s/[^\w'-]/ /g;
  
  my @words = split ' ', $text;
  for (@words){
    s/^['-]//g;
    s/['-]s?$//;
    next if length() < $min or length() > $max;
    next if exists $stop->{$_};
    next if /\d/ and not /^[12]\d{3}s?$/;
    next if /--/;
    push @{$words_all->{$_}}, $file_key;    
  }
}    

sub get_stop{
  # sample
  return {
    qw(
      and ''
      any ''
      the ''
      they ''
    )
  };
}

sub get_punc{
  # sample
  return {
    '&rsquo;' => undef,
    '&lsquo;' => undef,
    '&rdquo;' => undef,
    '&ldquo;' => undef,
  };
}

__DATA__
---------- Capture Output ----------
> "C:\Perl\bin\perl.exe" _new.pl

hyphen
cat
ad-hoc
O'Reilly
broken
dogs

> Terminated with exit code 0.
[download]

At the moment the full app is run locally on a copy of the web site.

It's generating 42k words but I'm working on the stop file to try and bring it down.

If this turns out to be fairly stable I'm considering compiling the regexes outside of the loop.

What do you reckon?

winxp, activestate 5.8

Update:
Corrected get_stop() sub