Just put a normal Perl expression to the pattern with /e.

$xml =~ s/(\<tag\>)(\d{11})(\<\/tag\>)/ $1 . md5_hex($2) . $3 /eg;
[download]

Update: Or, use a proper XML handling tool:

{   package XML::XSH2::Map;
    use Digest::MD5 qw( md5_hex );
}


use XML::XSH2;
xsh << '__XSH__';
open 1.xml;
for my $n in //num[xsh:matches(., '^[0-9]{11}$')]/text() {
    my $s = string($n);
    insert text { md5_hex($s) } replace $n ;
}
ls / ;
__XSH__
[download]