#!/usr/bin/perl -wl
use strict;

# define a HoH containing info about our external scripts
# "file" - the name of the file
# "timeout" - how long we wait for it to finish before giving up
# "tries" - how many attempts we make at exceuting it
# "sleepy" - to simulate an actual run and generate some errors
# "execute" - whether or not we execute it on this run

my %scripts = (
    1   => {
            file    => "/usr/bin/feed_the_cat.sh",
            timeout => 5,
            tries   => 2,
            sleepy  => 6, # will cause this one to fail
        },
    2   => {
            file    => "/usr/bin/put_out_the_garbage.sh",
            timeout => 10,
            tries   => 3,
            sleepy  => 5, # this one will succeed
        },
    3   => {
            file    => "/usr/bin/switch_off_the_lights.sh",
            timeout => 10,
            tries   => 2,
            sleepy  => 13, # this one will fail
    },
    # etc, etc...
);

# Set everything to initially be executed
for (keys %scripts) { $scripts{$_}{execute}++; }

# We write any failures to here
my $last_run_failed = "failed.txt";

# Were there any failures on the last run?
if (-f $last_run_failed) {
    print "Whoops, looks like we had a few failures on the last run - let's retry them :)";

    # Set all scripts to NOT be executed, then
    # reset for just those that failed last time
    for (keys %scripts) { $scripts{$_}{execute}=undef; }
    open FAILED, "<", $last_run_failed
        or die "Duh, I couldn't open $last_run_failed!:$!";
    while (<FAILED>) {
        chomp;
        $scripts{$_}{execute}++;
    }
    close FAILED;

    # Zap the failed file
    unlink $last_run_failed
        or die "Oops, I couldn't delete $last_run_failed";
}


SCRIPT:
for my $script (sort keys %scripts) {

    # Skip this one if we need to
    next SCRIPT if !defined $scripts{$script}{execute};
    my $tries = $scripts{$script}{tries};
    my $timeout = $scripts{$script}{timeout};
    my $sleepytime = $scripts{$script}{sleepy};

    TRY:
    for my $try (1 .. $tries) {
        print "Trying $scripts{$script}{file} (attempt $try/$tries).. ";

        eval {
            local $SIG{ALRM} = sub { die "alarm\n" };
            alarm $timeout;
            # The next line is just to simulate the scipt running
            # In the real world, you have something like :
            # system($scripts{$script}{file};
            sleep $sleepytime;
            alarm 0;
        };

        if ($@) {
            # It timed out
            print "whoops, timed out - let's try again...";
            next TRY;
        }
        else {
            # It worked
            print "Yup, that one worked - lets move onto the next one..";
            next SCRIPT;
        }
    }

    # If we get to here, the script has failed to return within the timeout
    # period after the specified number of tries
    # Therefore, we record the fact in the "failed" file
    print "Looks like $scripts{$script}{file} is foo-bar, giving giving up";
    open FAILED, ">>", $last_run_failed
        or die "Could not open $last_run_failed:$!";
    print FAILED "$script";
    close FAILED;
}

##</code><code>##

Trying /usr/bin/feed_the_cat.sh (attempt 1/2)..
whoops, timed out - let's try again...
Trying /usr/bin/feed_the_cat.sh (attempt 2/2)..
whoops, timed out - let's try again...
Looks like /usr/bin/feed_the_cat.sh is foo-bar, giving giving up
Trying /usr/bin/put_out_the_garbage.sh (attempt 1/3)..
Yup, that one worked - lets move onto the next one..
Trying /usr/bin/switch_off_the_lights.sh (attempt 1/2)..
whoops, timed out - let's try again...
Trying /usr/bin/switch_off_the_lights.sh (attempt 2/2)..
whoops, timed out - let's try again...
Looks like /usr/bin/switch_off_the_lights.sh is foo-bar, giving giving up

##</code><code>##

1
3

##</code><code>##

Whoops, looks like we had a few failures on the last run - let's retry them :)
Trying /usr/bin/feed_the_cat.sh (attempt 1/2)..
whoops, timed out - let's try again...
Trying /usr/bin/feed_the_cat.sh (attempt 2/2)..
whoops, timed out - let's try again...
Looks like /usr/bin/feed_the_cat.sh is foo-bar, giving giving up
Trying /usr/bin/switch_off_the_lights.sh (attempt 1/2)..
whoops, timed out - let's try again...
Trying /usr/bin/switch_off_the_lights.sh (attempt 2/2)..
whoops, timed out - let's try again...
Looks like /usr/bin/switch_off_the_lights.sh is foo-bar, giving giving up

##</code><code>##

Whoops, looks like we had a few failures on the last run - let's retry them :)
Trying /usr/bin/feed_the_cat.sh (attempt 1/2)..
Yup, that one worked - lets move onto the next one..
Trying /usr/bin/switch_off_the_lights.sh (attempt 1/2)..
Yup, that one worked - lets move onto the next one..

##</code><code>##

/bin/ls: failed.txt: No such file or directory