Here's what I'm using to recursively check and correct reference counts before deletion:

void Reference_recursivePurge (SV *ref, char *txt) {
    char label[1024];
    int rc;

    if (!SvROK(ref)) {
        rc = SvREFCNT(ref);
        if (rc > 1) {
            fprintf(stderr,"REFCOUNT for %s: %d\n", txt, rc);
            while (rc > 1) {
                SvREFCNT_dec(ref);
                rc--;
            }
        }
        return;
    }

    if (SvTYPE(SvRV(ref)) == SVt_PVAV) {
        AV *ray = (AV*)SvRV(ref);
        SV **cur;
        int i, len = av_len(ray);
        for (i = 0; i <= len; i++) {
            cur = av_fetch(ray, i, 1);
            sprintf(label, "%s[%d]", txt, i);
            Reference_recursivePurge(*cur, label);
            i++;
        }
        rc = SvREFCNT(ray);
        if (rc > 1) {
            fprintf(stderr,"REFCOUNT for %s: %d\n", txt, rc);
            while (rc > 1) {
                SvREFCNT_dec((SV*)ray);
                rc--;
            }
        }
        return;
    }

    if (SvTYPE(SvRV(ref)) == SVt_PVHV) {
        HV *hash = (HV*)SvRV(ref);
        SV *cur;
        char *key;
        int len;
        hv_iterinit(hash);
        while ((cur = hv_iternextsv(hash, &key, &len))) {
            sprintf(label, "%s->%s", txt, key);
            Reference_recursivePurge(cur, label);
        }
        rc = SvREFCNT(hash);
        if (rc > 1) {
            fprintf(stderr,"REFCOUNT for %s: %d\n", txt, rc);
            while (rc > 1) {
                SvREFCNT_dec((SV*)hash);
                rc--;
            }
        }
    }
}
[download]

Manually decrementing the count is a temporary hack solution; the reporting provides a tidy clue, then the idea here is to isolate the problem and see if getting the structure in question free'd via decrement solves the memory use issue.