my @subcategories = ("Physics");
my %unique_pages;

while (@subcategories) {
  my $next_subcategory = shift @subcategories;
  my @nodes = fetch_subcategory($next_subcategory);
  NODE: foreach my $node (@nodes) {
    if ($node->type() eq "subcategory") {
      push @subcategories, $node;
      next NODE;
    }
    $unique_pages{$node->unique_id()} = $node;
  }
}

foreach my $page (values %unique_pages) {
  my $page_content = fetch_page($page);
  # ... stuff here ...
}
[download]

In POE, you might keep the subcategories list and the unique pages hash in a session's heap. Class-scoped lexicals are also good here, especially if you plan for many sessions to share the burden of crawling. Here's one way to do it:

sub fetch_next_subcategory {
  my ($kernel, $heap) = @_[KERNEL, HEAP];
  my $next_subcategory = shift @{$heap->{subcategories}};
  if (defined $next_subcategory) {
    # ... start a subcategory fetch ...
  }
  else {
    # switch to fetching articles
    $kernel->yield("fetch_next_article");
  }
}

sub got_a_subcategory_page {
  my ($kernel, $heap, $page) = @_[KERNEL, HEAP, ARG0];
  my @nodes = parse_subcategory($page);
  NODE: foreach my $node (@nodes) {
    if ($node->type() eq "subcategory") {
      push @{$heap->{subcategories}}, $node;
    }
    else {
      push @{$heap->{pages}}, $node;
    }
  }
  $kernel->yield("fetch_next_subcategory");
}

sub fetch_next_article {
  my ($kernel, $heap) = @_[KERNEL, HEAP];

  my $next_page;
  while (defined($next_page = shift @{$heap->{pages}})) {
    last unless exists $heap->{unique_pages}{$next_page->unique_id()};
  }

  unless (defined $next_page) {
    # ... done.
    return;
  }

  $heap->{unique_pages}{$next_page->unique_id()} = 1;
  # ... start fetching next page
}

sub got_an_article_page {
  my ($kernel, $heap) = @_[KERNEL, HEAP];
  # ... process page.
  $kernel->yield("fetch_next_article");
}
[download]