which i don't want to make a code just to escape them all.

Why not? It's just a question of using quotemeta($1) instead of $1.

my $re_pat = '';
for ($glob_pat) {
    /\G ([^?*\\]+) /xsgc && $re_pat .= quotemeta($1);

    /\G \?    /xsgc && do { $re_pat .= '.';           redo };
    /\G \*    /xsgc && do { $re_pat .= '.*';          redo };
    /\G \\(.) /xsgc && do { $re_pat .= quotemeta($1); redo };

    /\G \Z /xsgc or redo;
}
[download]