Not only can it be done, that's exactly what Devel::PeekPoke::PP does.

$ perl -E'
    use Config qw( %Config );
    my $ptr_size = $Config{ptrsize};
    my $ptr_format =
       $ptr_size == 4 ? "L" :
       $ptr_size == 8 ? "Q" :
       die("Unrecognized pointer size");

    sub pv { unpack($ptr_format, pack("p", $_[0])) }
    sub peek { unpack('P'.$_[1], pack($ptr_format, $_[0])) }

    my $s = "abcdef";
    my $ptr = pv($s);
    say peek($ptr+1, 4);
'
bcde
[download]