Your code as presented in the OP only filters on the first column, so that's the one to use in this pre-filter - if that's the way you want to go with this.

#!/usr/bin/env perl
use strict;
use warnings;

use Test::More tests => 3;
use Test::NoWarnings;

my @matrix = (
    [1, 2],
    [undef, 9],
    [3, 4]
);

my $to_keep = 3;

my @matrix_defined = grep { defined $_->[0] } @matrix;
is $#matrix_defined, 1, '2 rows after prefilter';

my @matrix_filtered = grep { $_->[0] =~ /$to_keep/ } @matrix_defined;
is $#matrix_filtered, 0, '1 row after full filter';
[download]