Your code as presented in the OP only filters on the first column, so that's the one to use in this pre-filter - if that's the way you want to go with this.
#!/usr/bin/env perl
use strict;
use warnings;
use Test::More tests => 3;
use Test::NoWarnings;
my @matrix = (
[1, 2],
[undef, 9],
[3, 4]
);
my $to_keep = 3;
my @matrix_defined = grep { defined $_->[0] } @matrix;
is $#matrix_defined, 1, '2 rows after prefilter';
my @matrix_filtered = grep { $_->[0] =~ /$to_keep/ } @matrix_defined;
is $#matrix_filtered, 0, '1 row after full filter';