@$row_data{groups} = grep{ $_->{whatever} == $foo } @$row_data{groups} ;

##</code><code>##

for( qw/groups fields/ ){
  my $aref = $row_data{$_};
  @$aref = grep{ $_->{whatever} == $foo } @$aref;
}