while( my $file = <graph_set_*.out> ) {
    next unless $file =~ m[graph_set_[123]?[0-9].out];
    open IN, '<', $file 
        or warn "Couldn't open file $file because $!";
    #Do stuff
    close IN;
}

##</code><code>##

for $file (sort{ $a=~/(\d+)/ <=> $b=~/(\d+)/ } <graph_set_*.out> ) {
    open...
    ....
    close...
}