As you wrote this I was actually fiddling with exactly that :-) The relevant Perl source appears to be pp_open_dir in pp_sys.c, which uses the IoDIRP macro, which apparently accesses the DIR * xiou_dirp slot of struct xpvio, but I can't seem to find any more documentation on it.

Disclaimer: I am not an XS expert, I can't guarantee that the following is entirely correct! I got some of this from the Inline::C::Cookbook, a bit of research in perlapi, and a bit of fiddling...

myreaddir does all of the work of opening and reading the directory in C, returning a Perl list, while _xs_myfdopendir with the Perl wrapper myfdopendir attempts to be a custom opendir.

use warnings;
use strict;

use Inline C => <<'END_OF_C';
void myreaddir(SV* sv_dirn) {
    Inline_Stack_Vars;
    Inline_Stack_Reset;
    int fd = open( SvPVx(sv_dirn, PL_na),
        O_RDONLY|O_DIRECTORY|O_NOFOLLOW);
    if (fd<0) Inline_Stack_Return(0);
    DIR* dir = fdopendir(fd);
    if (dir==NULL) Inline_Stack_Return(0);
    struct dirent *dp;
    while ( (dp=readdir(dir)) != NULL )
        Inline_Stack_Push(sv_2mortal(
            newSVpvf("%s", dp->d_name) ));
    if( closedir(dir)!=0 ) Inline_Stack_Return(0);
    Inline_Stack_Done;
}
int _xs_myfdopendir(SV* sv_dirn, SV* sv_hnd) {
    int fd = open( SvPVx(sv_dirn, PL_na),
        O_RDONLY|O_DIRECTORY|O_NOFOLLOW);
    if (fd<0) return 0;
    DIR* dir = fdopendir(fd);
    if (dir==NULL) return 0;
    IoDIRP(sv_2io(sv_hnd)) = dir;
    return 1;
}
END_OF_C
use Symbol qw/geniosym/;
use File::Spec;
sub myfdopendir {
    return unless _xs_myfdopendir(
        $_[0]//File::Spec->curdir,
        my $dh=geniosym );
    return $dh;
}

use Data::Dump;
my @x = myreaddir('/tmp') or die $!;
dd @x;
my $dh = myfdopendir('/tmp') or die $!;
dd readdir $dh;
closedir $dh or die $!;
[download]

Update: A couple of Perl modules that use XS to read directories, in particular the first one's readdir_hashref looks like it could be modified fairly simply: ReadDir, IO-Dirent, PerlIO-Util