Consider to choose another data representation.
You can tie numbers and priorities in some ways

# as array of arrays
my @data_aoa = (
  [1, 5],                # [number, priority]
  [5, 2],
  [3, 1]
); 

# or more verbose as array of hashes
my @data_aoh = (
  {number => 1, pri => 5 },
  {number => 5, pri => 2 },
  {number => 3, pri => 1 }
);

# now sort by priority

my @sorted_aoa = sort { $a->[1] <=> $b->[1] } @data_aoa;

my @sorted_aoh = sort { $a->{pri} <=> $b->{pri} } @data_aoh;
[download]