Event listener is a class which loops over the Thread::Queue contents (listen method) and based on the contents inserts data in database and does other processing. Crawler class provides methods to crawl over the web sites (using WWW::Mechanize) and adds data to Queue. Crawler first login to the website and collect various links from where data is to be extracted and then inside a infinite loop crawl over those pages. Crawler returns immediately if there is no data available to extract, if data is found then it stays there and refreshes the page every 30 sec to get data till the data is available.

#!c://perl/bin/perl -w

use strict;
use warnings;

use threads;
use threads::shared;
use Thread::Queue;

use App::Options;

use Crawler;
use EventListner;

use Data::Dumper;

my $q = Thread::Queue->new();

my $options = \%App::options; # Get command line options

my $page_status = {};
share $page_status;
my $page_priorty = {};
share $page_priorty;

# Open Listneracks
my $event_listner = EventListner->new($q, $page_status, $page_priorty)
+;
my $listner_thr = threads->create(sub { $event_listner->listen(); });
$listner_thr->detach();

# Create crawler object
my $crawler = Crawler->new($q, $options);
$crawler->login();          # Login
$crawler->fetch_pages();    # Fetch pages

my $threads_count :shared = 0;
while (1) {
    # Get all closed pages and sort on there priorty
    my @closed_pages = grep { $page_status->{$_} eq 'C' } keys %$page_
+status;
    my @priorty_sort_pages = sort {$page_priorty->{$a} <=> $page_prior
+ty->{$b}} @closed_pages;

    foreach my $page (@priorty_sort_pages) {
        if ($threads_count < $options->{max_crawlers}) {
            $threads_count++;
            $page_status->{$page} = 'O';
            $page_priorty->{$page}++;
            my $thr = threads->create(sub {
                threads->detach();
                $crawler->run($page);
                $crawler->reset();
                unless ($page_status->{$page} eq 'X') {
                    $page_status->{$page} = 'C';
                }
                $threads_count--;
            });
            #if ($thr->is_running()) {
                sleep($options->{pause}) if $options->{pause};
            #}
            #print "A\n";
        }
        #print "B\n";
    }
}
[download]

• the script crashes only if no data is found on the first two pages(please note that crawler returns if no data is found.) but even this is not consistent. (Some time it works even in this conditions but it fails under no other conditions)
• I did some debugging by adding some print statements and found it prints "A" but crashes before "B" is printed.