my %group2user;
foreach my $rec (@all_groups) {
  push @{$group2user{$rec->[0]}}, $rec;
}
foreach my $group_id (sort keys %group2user) {
  # Do whatever
}