It looks like you are dealing with XML. It's generally a bad idea to try and hand parse XML. However there are some really good modules around to help. Consider this XML::Twig solution:

use strict;
use warnings;
use XML::Twig;

my $xml = <<XML;
<data>
    <ignore>
    This should be ignored.
    </ignore>
    <address>
    <state>address1</state>
    <ref refid="aff1">1</ref>
    <ref refid="aff2">2</ref>
    <ref refid="e2"/>
    <ref refid="e4"/>
    <ref refid="aff4">4</ref>
    </address>
    <address>
    <state>address1</state>
    <ref refid="aff1">1</ref>
    <ref refid="aff2">2</ref>
    <ref refid="e4"/>
    <ref refid="aff3">3</ref>
    </address>
</data>
XML

my $twig = XML::Twig->new ( 
    twig_roots   => { 'address' => \&sortAddress, },
    twig_print_outside_roots => 1,
    pretty_print => 'indented',
    );

$twig->parse ($xml);


sub sortAddress {
    my ($t, $address)= @_;
    my @children = $address->cut_children ();
    my @untouchables = grep {$_->tag () ne 'ref'} @children;
    my @sorted =
        sort {$a->att ('refid') cmp $b->att ('refid')}
        grep {$_->tag () eq 'ref'} @children;
    
    $_->paste (last_child => $address) for @untouchables;
    $_->paste (last_child => $address) for @sorted;
    $address->print ();
  }
[download]

Prints:

<data>
    <ignore>
    This should be ignored.
    </ignore>
    
  <address>
    <state>address1</state>
    <ref refid="aff1">1</ref>
    <ref refid="aff2">2</ref>
    <ref refid="aff4">4</ref>
    <ref refid="e2"/>
    <ref refid="e4"/>
  </address>
    
  <address>
    <state>address1</state>
    <ref refid="aff1">1</ref>
    <ref refid="aff2">2</ref>
    <ref refid="aff3">3</ref>
    <ref refid="e4"/>
  </address>
</data>
[download]