sub change_field_to_check {
  my $hoa = shift;
  my $field = shift;
  $field -= 1;

  my $new_arrangement;

  foreach (keys %$hoa) {
    $new_arrangement->{ $hoa->{ $_ }[ $field ] } = $hoa->{ $_ };
  }

  return $new_arrangement;
}

$file_A = change_field_to_check( $file_A, 4 ); # Field 4
$file_B = change_field_to_check( $file_B, 4 ); # Field 4

foreach (keys %$file_A) {
  delete $file_A->{ $_ } unless exists( $file_B->{ $_ } );
}