#!/usr/bin/perl
use strict;
use warnings;

use LWP::RobotUA;
use HTML::LinkExtor;
use HTML::TokeParser;
use URI::URL;

use Data::Dumper; # for show and troubleshooting

my $url = "http://www.phpBBhacks.com/forums/viewforum.php?f=17";
my $ua = LWP::RobotUA->new;
my $lp = HTML::LinkExtor->new(\&wanted_links);

my @links;
get_threads($url);

foreach my $page (@links) { # this loops over each link collected from the index
	my $r = $ua->get($page);
	if ($r->is_success) {
		my $stream = HTML::TokeParser->new(\$r->content) or die "Parse error in $page: $!";
		# just printing what was collected
		print Dumper get_thread($stream);
		# would instead have database insert statement at this point
	 } else {
		warn $r->status_line;
	 }
}

sub get_thread {
	my $p = shift;
	my ($title, $name, @thread);
	while (my $tag = $p->get_tag('a','span')) {
		if (exists $tag->[1]{'class'}) {
			if ($tag->[0] eq 'span') {
				if ($tag->[1]{'class'} eq 'name') {
					$name = $p->get_trimmed_text('/span');
				} elsif ($tag->[1]{'class'} eq 'postbody') {
					my $post = $p->get_trimmed_text('/span');
					push @thread, {'name'=>$name, 'post'=>$post};
				}
			} else {
				if ($tag->[1]{'class'} eq 'maintitle') {
					$title = $p->get_trimmed_text('/a');
				}
			}
		}
	}
	return {'title'=>$title, 'thread'=>\@thread};
}

sub get_threads {
	my $page = shift;
	my $r = $ua->request(HTTP::Request->new(GET => $url), sub {$lp->parse($_[0])});
	# Expand URLs to absolute ones
	my $base = $r->base;
	return [map { $_ = url($_, $base)->abs; } @links];
}

sub wanted_links {
	my($tag, %attr) = @_;
	return unless exists $attr{'href'};
	return if $attr{'href'} !~ /^viewtopic\.php\?t=/;
	push @links, values %attr;
}


##</code><code>##


$VAR1 = {
          'thread' => [
                        {
                          'post' => 'Hello, I\'m pretty new to PHPNuke. I\'ve got my site up and running great! I\'m now starting to make modifications, add modules etc. I\'m using the most recent RavenPHP76. I want to display the 5 most recent forum posts at the top of the forum page. I\'m not sure if this functionality is built in, if so, how to activate. Or if there is a module or block made to do this. I looked at Raven\'s Collapsing Forum block but wasn\'t crazy about the format, and I don\'t want it to be collapsable. Thanks! mopho',
                          'name' => 'mopho'
                        },
                        {
                          'post' => 'hi there',
                          'name' => 'sail'
                        },
                        {
                          'post' => 'thanks for asking this; :not very sure if i got you right; Do you want to have a feed of the last forumthreads? guess the easiest way is to go to raven and ask how he did it. hth sail.',
                          'name' => 'sail'
                        },
                        {
                          'post' => 'Thanks. i found what I was looking for. It wasn\'t so easy to find! It\'s called glance_mod. mopho',
                          'name' => 'mopho'
                        },
                        {
                          'post' => 'hi there thx',
                          'name' => 'sail'
                        },
                        {
                          'post' => 'it sound interesting - i will have also a look i google after it - and try to find out more regards sailor',
                          'name' => 'sail'
                        }
                      ],
          'title' => 'Recent Forum Posts Module'
        };