my %config_files = ( cluster1 => ['1h1', 'h2', 'h3', 'h4', 'h5', 'h6', '1h7'], cluster2 => ['2h1', 'h2', 'h3', 'h4', 'h5', 'h6', '2h7'], cluster3 => ['3h1', 'h2', 'h3', 'h4', 'h5', 'h6', '3h7'], cluster4 => ['4h1', 'h2', 'h3', 'h4', 'h5', 'h6', '4h7'], cluster5 => ['5h1', 'h2', 'h3', 'h4', 'h*5', 'h6', '5h7'], ); my @order = sort keys %config_files; print "Order will be @order\n"; for my $cluster (@order) { print "Cluster is $cluster\n"; my @machines = @{$config_files{$cluster}}; my $last = pop @machines; for my $box (@machines) { &start_up($box) or &failed($cluster, $box); } &special($last); } print "Done\n"; exit; sub start_up { my $box = shift; return 0 if $box eq 'h*5'; # test failed routine # do stuff, return 1 for success 0 or undef for failure print "\tStarted $box\n"; 1; } sub special { # do box 7 stuff my $box = shift; print "\tSpecial $box\n"; 1; } sub failed { my ($cluster, $box) = @_; # do whatever you want like retry until sucess # then you return to loop and continue my $tries = 5; my $delay = 2; while ($tries) { $tries--; sleep $delay; print "Retrying $box in $cluster\n"; return if &start_up($box); } die "Unable to start $box in $cluster\n"; }