#!perl
# SMiTZ's Big Bad Support Crawler
# crawls support.intel.com, reading title and meta description tags,
# storing these in a lovely database.
#
# v0.1

use strict;
use warnings;
use WWW::Robot;
use DBI;
use Data::Dumper;

my $docRoot = 'http://support.intel.com';
my $databaseName = 'spider1';
my $databaseTable = 'index';

# Connect to DB

my $dbh = DBI->connect("DBI:ODBC:$databaseName", {RaiseError=>1}) or d
+ie ("Couldn\'t connect to database: $DBI::errstr; stopped");

my $robot = new WWW::Robot(    'NAME' =>    'SMiTZ\'s Bot',
                'VERSION' =>    0.1,
                'EMAIL' =>    '***@intel.com',
                'VERBOSE' =>    0,        #NAUGHTY BOY!!! Change ASAP
                'DELAY' =>    0,
                'IGNORE_TEXT' =>    0);        .
$robot->proxy('http', 'http://***.***.intel.com:911');    # damn firew
+all

$robot->addHook('follow-url-test', \&follow_url_test);
$robot->addHook('invoke-on-contents', \&invoke_on_contents);

$robot->run($docRoot);
$dbh->disconnect();



#---------------------------------------------------------------------
+----------------------
# Hook subs

sub follow_url_test {
    my ($robot, $hook, $url) = @_;
    return 0 unless $url->scheme eq 'http';
    return 0 if $url =~ /\.(gif|jpg|png|xbm|au|wav|mpg|doc|xml|ppt)$/;
    return $url =~ /^$docRoot/;
    
}

sub invoke_on_contents {
    my ($robot, $hook, $url, $response, $structure) = @_;
    return unless $response->content_type eq 'text/html';
    my $desc = $response->header("X-meta-keywords");
    $desc = 'none, none, none, none, none' unless ($desc);
    my $title = $response->header("title");
    my @desc = split(/,/, $desc);
    $dbh->do(q{
        INSERT INTO index (url, title, description1, description2, des
+cription3, description4, description5)
        VALUES (?, ?, ?, ?, ?, ?, ?)
    }, undef, $url, $title, $desc[0], $desc[1], $desc[2], $desc[3], $d
+esc[4]) 
        or die $dbh->errstr;
    print "*";
    
}
[download]