Others beat me to it but here's a benchmark.

use strict;
use warnings;

use 5.014;

use Benchmark qw{ cmpthese };

my @letters = ( q{a} .. q{z} );
my $raTest  = [ qw{ a b a } ];
my $raReal  = [ map { $letters[ rand @letters ] } 1 .. 100000 ];

my $rhRes1 = wollmers( $raTest );
my $rhRes2 = johngg( $raTest );

say do {
   local $" = q{, };
   qq{$_ => [ @{ $rhRes1->{ $_ } } ]}
   } for sort keys %$rhRes1;
say do {
   local $" = q{, };
   qq{$_ => [ @{ $rhRes1->{ $_ } } ]}
   } for sort keys %$rhRes2;

cmpthese( -10,
   {
      johngg   => sub { my $ret = johngg( $raReal ) },
      wollmers => sub { my $ret = wollmers( $raReal ) },
   }
   );

sub wollmers
{
    my $aref = shift;
    my $index;
    my $rhRet;
    for ( $index = 0; $index <= $#$aref; $index ++ )
    {
        push @{ $rhRet->{ $aref->[ $index ] } }, $index;
    }
    return $rhRet;
}

sub johngg
{
    my $aref = shift;
    my $idx = 0;
    my $rhRet;
    push @{ $rhRet->{ $_ } }, $idx ++ for @{ $aref };
    return $rhRet;
}
[download]

The output.

a => [ 0, 2 ]
b => [ 1 ]
a => [ 0, 2 ]
b => [ 1 ]
           Rate wollmers   johngg
wollmers 17.6/s       --     -26%
johngg   24.0/s      36%       --
[download]

A modest increase in speed so perhaps the Inline::C suggestion will be required.