sub test_url {
    my ( $s, $server ) = @_;
    # return 1;  # Ok to index/spider
    # return 0;  # No, don't index or spider;

    # ignore any common image files
    return 0 if $s =~ /\.(gif|jpg|jpeg|png)?$/;

    # ignore directory listing sorting links
    return 0 if $s =~ /\?(C=N;O=D|C=M;O=A)$/;

    # make sure that the path is limited to the docs path
    return $s =~ m[^/starteam_area/];
}

my $res;
$res = test_url('http://someurl.com/?C=N;O=D');
print "returned value was - ".$res."\n";
$res = test_url('http://someurl.com/?C=M;O=A');
print "returned value was - ".$res."\n";
$res = test_url('http://someurl.com/');
print "returned value was - ".$res."\n";
$res = test_url('http://someurl.com/?C=X;O=A');
print "returned value was - ".$res."\n";

----
#output

returned value was - 0
returned value was - 0
returned value was - 
returned value was -