As written your delay is in the child fork. Instead you need to delay in the parent fork:

#!/usr/software/bin/perl5.8.8 -w

use strict;
use warnings;
use Cwd qw(abs_path);
use Parallel::ForkManager;

my $pm = new Parallel::ForkManager(8);
my $tasks = 3;

$pm->run_on_finish(
    sub {
        my ($pid, $exit_code) = @_;
        my $output = "$pid has finished with exit code $exit_code.\n";
        print $output;
    }
    );

for my $task (1 .. $tasks) {
    if ($pm->start($task)) {
        sleep 120 if $task != $tasks;
        next;
    }

    # This is the forked process
    my $time = time;
    print "$task\n";
    print "$time\n";
    $pm->finish();
}

$pm->wait_all_children;
[download]

Note that I haven't tested the code.