while (<$ref_fh>) {
    chomp;
    ++$ref_left{$_};
}
...
while (<$data_fh>) {
    my ($left, $right) = split ' ', $_, 2;
    next unless $ref_left{$left} or $ref_right{$right};
    ++$ref_right{$right};
}