How about:

use warnings;
use strict;


my @test = qw(1 12 123 1234 12345 1x12x123x1234x12345);
my $sep = '_';


for my $s (@test) {
    my $sa = append($s, $sep);
    if (defined($sa)) {
        print "'$s' -> '$sa' \n";
        }
    else {
        print "'$s' append undefined \n";
        }
    }


sub append {

    my ($string,
        $sep,
        ) = @_;

    return unless  # return undef unless substitution(s) made
        $string =~ s{ (?<! \d) (\d{2,4}) (?! \d)   }
                    { join $sep, split /\B/, $1, 3 }xmsge;

    return $string;  # return string with substitution(s)

    }
[download]

Output:

'1' append undefined
'12' -> '1_2'
'123' -> '1_2_3'
'1234' -> '1_2_34'
'12345' append undefined
'1x12x123x1234x12345' -> '1x1_2x1_2_3x1_2_34x12345'
[download]