sub test_url {
    my ( $uri, $server ) = @_;
    # return 1;  # Ok to index/spider
    # return 0;  # No, don't index or spider;

    # ignore any common image files
    return 0 if $uri->path =~ /\.(gif|jpg|jpeg|png)?$/;

    # ignore directory listing sorting links
    return 0 if $uri->path =~ /\?(C=N;O=D|C=M;O=A)$/;

    # make sure that the path is limited to the docs path
    return $uri->path =~ m[^/starteam_area/];
}