0: Nothing special, just some Friday night excitment, a script to take a apache referer log and print some information about the search engines and terms that people are using to reach your website. 
1: 
2: I'd be curious about better ways to do this & existing scripts that do this better....
3: 
4: #!/usr/local/bin/perl -w
5: 
6: use strict;
7: 
8: #ref.txt comes from:
9: # 'ls /var/log/apache/referr.*.gz | xargs zcat >> ~/ref.txt'
10: # 'cat /var/log/apache/referer.log >> ~/ref.txt'
11: 
12: open FH,"./ref.txt" || die "$!";
13: my @lines=<FH>;
14: close (FH);
15: 
16: # none of these are search engines
17: my $ignore_hosts=
18:     qr(\Qlctc.org\E|\Qgradetheprof.net\E|\Q10.0.0\E|perlmonks);
19: #   qr((\Qlctc.org\E)|(\Qgradetheprof.net\E)|(\Q10.0.0\E)|(perlmonks));
20: 
21: 
22: my $garbage=
23:   qr(%..|.=);
24: 
25: my %hosts;
26: my $search_phrase;
27: my %search_phrases;
28: my %search_words;
29: my @words;
30: my $word;
31: 
32: foreach (@lines){
33: 	# if there is a query string
34: 	# and it isn't from our CGI
35: 	if ((m/\?/) && (m/\+/) 	&& ($_!~m/$ignore_hosts/) ){
36: 		m/
37: 			(http:..) 
38: 			([a-z.]*)	#hostname
39: 			(.*\?)	 	#bit before query string
40: 	 		(.*)		# search string
41: 			(-\>.*)		# page refered to
42: 		/xi;
43: 
44: 
45: 		$hosts{$2}++;
46: 
47: 		$search_phrase=$4;
48: 		$search_phrase=~s/$garbage//g;
49: 		$search_phrase=~s/&.*$//;
50: 		$search_phrase=~s/\+/ /g;
51: 		$search_phrase=lc($search_phrase);
52: 
53: 		$search_phrases{$search_phrase}++;
54: 
55: 		@words=split(/ /,$search_phrase);
56: 		foreach $word (@words){
57: 			$search_words{$word}++;
58: 		}		
59: 		
60: 	}
61: }
62: 
63: foreach (sort (keys %hosts)){
64: 	print "$hosts{$_} searches from $_\n";
65: }
66: 
67: 
68: 
69: print "\n search words:\n";
70:   foreach my $key (sort { $search_words{$b} <=> $search_words{$a} } keys %search_words) {
71:       print  "$search_words{$key} $key\n";
72:   }
73: 
74: print "\n search phrases:\n";
75:   foreach my $key (sort { $search_phrases{$b} <=> $search_phrases{$a} } keys %search_phrases) {
76:       print  "$search_phrases{$key} $key\n";
77:   }