pycore_stackref.h

#ifndef Py_INTERNAL_STACKREF_H
#define Py_INTERNAL_STACKREF_H
#ifdef __cplusplus
extern "C" {
#endif

#ifndef Py_BUILD_CORE
#  error "this header requires Py_BUILD_CORE define"
#endif

#include "pycore_object.h"        // Py_DECREF_MORTAL
#include "pycore_object_deferred.h" // _PyObject_HasDeferredRefcount()

#include <stdbool.h>              // bool


/*
  This file introduces a new API for handling references on the stack, called
  _PyStackRef. This API is inspired by HPy.

  There are 3 main operations, that convert _PyStackRef to PyObject* and
  vice versa:

    1. Borrow (discouraged)
    2. Steal
    3. New

  Borrow means that the reference is converted without any change in ownership.
  This is discouraged because it makes verification much harder. It also makes
  unboxed integers harder in the future.

  Steal means that ownership is transferred to something else. The total
  number of references to the object stays the same.  The old reference is no
  longer valid.

  New creates a new reference from the old reference. The old reference
  is still valid.

  All _PyStackRef must be operated on by the new reference operations:

    1. DUP
    2. CLOSE

   DUP is roughly equivalent to Py_NewRef. It creates a new reference from an old
   reference. The old reference remains unchanged.

   CLOSE is roughly equivalent to Py_DECREF. It destroys a reference.

   Note that it is unsafe to borrow a _PyStackRef and then do normal
   CPython refcounting operations on it!
*/

#define Py_INT_TAG 3
#define Py_TAG_INVALID 2
#define Py_TAG_REFCNT 1
#define Py_TAG_BITS 3

#define Py_TAGGED_SHIFT 2

#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)

PyAPI_FUNC(PyObject *) _Py_stackref_get_object(_PyStackRef ref);
PyAPI_FUNC(PyObject *) _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber);
PyAPI_FUNC(_PyStackRef) _Py_stackref_create(PyObject *obj, uint16_t flags, const char *filename, int linenumber);
PyAPI_FUNC(void) _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber);
PyAPI_FUNC(_PyStackRef) _Py_stackref_get_borrowed_from(_PyStackRef ref, const char *filename, int linenumber);
PyAPI_FUNC(void) _Py_stackref_set_borrowed_from(_PyStackRef ref, _PyStackRef borrowed_from, const char *filename, int linenumber);
extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref);

static const _PyStackRef PyStackRef_NULL = { .index = 0 };
static const _PyStackRef PyStackRef_ERROR = { .index = (1 << Py_TAGGED_SHIFT) };

#define PyStackRef_None ((_PyStackRef){ .index = (2 << Py_TAGGED_SHIFT) } )
#define _Py_STACKREF_FALSE_INDEX (3 << Py_TAGGED_SHIFT)
#define _Py_STACKREF_TRUE_INDEX (4 << Py_TAGGED_SHIFT)
#define PyStackRef_False ((_PyStackRef){ .index = _Py_STACKREF_FALSE_INDEX })
#define PyStackRef_True ((_PyStackRef){ .index = _Py_STACKREF_TRUE_INDEX })

#define INITIAL_STACKREF_INDEX (5 << Py_TAGGED_SHIFT)

#define PyStackRef_ZERO_BITS PyStackRef_NULL

static inline _PyStackRef
PyStackRef_Wrap(void *ptr)
{
    assert(ptr != NULL);
#ifdef Py_DEBUG
    assert(((uint64_t)ptr & Py_TAG_BITS) == 0);
    return (_PyStackRef){ .index = ((uint64_t)ptr) | Py_TAG_INVALID };
#else
    return (_PyStackRef){ .index = (uint64_t)ptr };
#endif
}

static inline void *
PyStackRef_Unwrap(_PyStackRef ref)
{
#ifdef Py_DEBUG
    assert ((ref.index & Py_TAG_BITS) == Py_TAG_INVALID);
    return (void *)(ref.index & ~Py_TAG_BITS);
#else
    return (void *)(ref.index);
#endif
}

static inline int
PyStackRef_RefcountOnObject(_PyStackRef ref)
{
    return (ref.index & Py_TAG_REFCNT) == 0;
}

static inline int
PyStackRef_IsNull(_PyStackRef ref)
{
    return ref.index == 0;
}

static inline bool
PyStackRef_IsError(_PyStackRef ref)
{
    return ref.index == (1 << Py_TAGGED_SHIFT);
}

static inline bool
PyStackRef_IsMalformed(_PyStackRef ref)
{
    return (ref.index & Py_TAG_BITS) == Py_TAG_INVALID;
}

static inline bool
PyStackRef_IsValid(_PyStackRef ref)
{
    /* Invalid values are ERROR and NULL */
    return !PyStackRef_IsError(ref) && !PyStackRef_IsNull(ref);
}

static inline int
PyStackRef_IsTrue(_PyStackRef ref)
{
    return _Py_stackref_get_object(ref) == Py_True;
}

static inline int
PyStackRef_IsFalse(_PyStackRef ref)
{
    return _Py_stackref_get_object(ref) == Py_False;
}

static inline int
PyStackRef_IsNone(_PyStackRef ref)
{
    return _Py_stackref_get_object(ref) == Py_None;
}

static inline bool
PyStackRef_IsTaggedInt(_PyStackRef ref)
{
    return (ref.index & Py_TAG_BITS) == Py_INT_TAG;
}

static inline PyObject *
_PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber)
{
    assert(!PyStackRef_IsError(ref));
    assert(!PyStackRef_IsTaggedInt(ref));
    _Py_stackref_record_borrow(ref, filename, linenumber);
    return _Py_stackref_get_object(ref);
}
#define PyStackRef_AsPyObjectBorrow(REF) _PyStackRef_AsPyObjectBorrow((REF), __FILE__, __LINE__)

static inline PyObject *
_PyStackRef_AsPyObjectSteal(_PyStackRef ref, const char *filename, int linenumber)
{
    PyObject *obj = _Py_stackref_close(ref, filename, linenumber);
    if (PyStackRef_RefcountOnObject(ref)) {
        return obj;
    }
    return Py_NewRef(obj);
}
#define PyStackRef_AsPyObjectSteal(REF) _PyStackRef_AsPyObjectSteal((REF), __FILE__, __LINE__)

static inline _PyStackRef
_PyStackRef_FromPyObjectNew(PyObject *obj, const char *filename, int linenumber)
{
    assert(obj != NULL);
    uint16_t flags = 0;
    if (!_Py_IsImmortal(obj)) {
        _Py_INCREF_MORTAL(obj);
    } else {
        flags = Py_TAG_REFCNT;
    }
    return _Py_stackref_create(obj, flags, filename, linenumber);
}
#define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj), __FILE__, __LINE__)

static inline _PyStackRef
_PyStackRef_FromPyObjectSteal(PyObject *obj, const char *filename, int linenumber)
{
    assert(obj != NULL);
    uint16_t flags = 0;
    if (_Py_IsImmortal(obj)) {
        flags = Py_TAG_REFCNT;
    }
    return _Py_stackref_create(obj, flags, filename, linenumber);
}
#define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj), __FILE__, __LINE__)

static inline _PyStackRef
_PyStackRef_FromPyObjectBorrow(PyObject *obj, const char *filename, int linenumber)
{
    return _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber);
}
#define PyStackRef_FromPyObjectBorrow(obj) _PyStackRef_FromPyObjectBorrow(_PyObject_CAST(obj), __FILE__, __LINE__)

/* Tag a PyObject pointer as a borrowed operand for BORROW variants. */
static inline uintptr_t
PyStackRef_TagBorrow(PyObject *obj)
{
    return (uintptr_t)obj | Py_TAG_REFCNT;
}

/* Strip tag bits from a pre-tagged operand to recover the PyObject pointer. */
static inline PyObject *
PyStackRef_UntagBorrow(PyObject *tagged)
{
    return (PyObject *)((uintptr_t)tagged & ~Py_TAG_BITS);
}

/* Create a stackref from a pre-tagged operand (tag bits already set).
   Used by _LOAD_CONST_INLINE_BORROW variants where the operand is
   tagged at trace creation time to avoid tagging on every execution. */
static inline _PyStackRef
_PyStackRef_FromPreTagged(PyObject *tagged, const char *filename, int linenumber)
{
    assert((uintptr_t)tagged & Py_TAG_REFCNT);
    PyObject *obj = (PyObject *)((uintptr_t)tagged & ~Py_TAG_BITS);
    return _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber);
}
#define PyStackRef_FromPreTagged(tagged) _PyStackRef_FromPreTagged(_PyObject_CAST(tagged), __FILE__, __LINE__)

static inline void
_PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber)
{
    assert(!PyStackRef_IsError(ref));
    assert(!PyStackRef_IsNull(ref));
    if (PyStackRef_IsTaggedInt(ref)) {
        return;
    }
    PyObject *obj = _Py_stackref_close(ref, filename, linenumber);
    assert(Py_REFCNT(obj) > 0);
    if (PyStackRef_RefcountOnObject(ref)) {
        Py_DECREF(obj);
    }
}
#define PyStackRef_CLOSE(REF) _PyStackRef_CLOSE((REF), __FILE__, __LINE__)

static inline void
_PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber)
{
    assert(!PyStackRef_IsError(ref));
    if (PyStackRef_IsNull(ref)) {
        return;
    }
    _PyStackRef_CLOSE(ref, filename, linenumber);
}
#define PyStackRef_XCLOSE(REF) _PyStackRef_XCLOSE((REF), __FILE__, __LINE__)

static inline _PyStackRef
_PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
{
    assert(!PyStackRef_IsError(ref));
    assert(!PyStackRef_IsNull(ref));
    if (PyStackRef_IsTaggedInt(ref)) {
        return ref;
    }
    PyObject *obj = _Py_stackref_get_object(ref);
    uint16_t flags = 0;
    if (PyStackRef_RefcountOnObject(ref)) {
        Py_INCREF(obj);
    } else {
        flags = Py_TAG_REFCNT;
    }
    _PyStackRef new_ref = _Py_stackref_create(obj, flags, filename, linenumber);
    if (flags == Py_TAG_REFCNT && !_Py_IsImmortal(obj)) {
        _PyStackRef borrowed_from = _Py_stackref_get_borrowed_from(ref, filename, linenumber);
        _Py_stackref_set_borrowed_from(new_ref, borrowed_from, filename, linenumber);
    }
    return new_ref;
}
#define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__)

static inline void
_PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber)
{
    assert(!PyStackRef_IsError(ref));
    assert(!PyStackRef_IsNull(ref));
    assert(!PyStackRef_IsTaggedInt(ref));
    PyObject *obj = _Py_stackref_close(ref, filename, linenumber);
    assert(Py_REFCNT(obj) > 0);
    if (PyStackRef_RefcountOnObject(ref)) {
        _Py_DECREF_SPECIALIZED(obj, destruct);
    }
}
#define PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT) _PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT, __FILE__, __LINE__)

static inline _PyStackRef
_PyStackRef_Borrow(_PyStackRef ref, const char *filename, int linenumber)
{
    assert(!PyStackRef_IsError(ref));
    assert(!PyStackRef_IsNull(ref));
    if (PyStackRef_IsTaggedInt(ref)) {
        return ref;
    }
    PyObject *obj = _Py_stackref_get_object(ref);
    _PyStackRef new_ref = _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber);
    if (!_Py_IsImmortal(obj)) {
        _Py_stackref_set_borrowed_from(new_ref, ref, filename, linenumber);
    }
    return new_ref;
}
#define PyStackRef_Borrow(REF) _PyStackRef_Borrow((REF), __FILE__, __LINE__)

#define PyStackRef_CLEAR(REF) \
    do { \
        _PyStackRef *_tmp_op_ptr = &(REF); \
        _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \
        *_tmp_op_ptr = PyStackRef_NULL; \
        PyStackRef_XCLOSE(_tmp_old_op); \
    } while (0)

static inline _PyStackRef
_PyStackRef_FromPyObjectStealMortal(PyObject *obj, const char *filename, int linenumber)
{
    assert(obj != NULL);
    assert(!_Py_IsImmortal(obj));
    return _Py_stackref_create(obj, 0, filename, linenumber);
}
#define PyStackRef_FromPyObjectStealMortal(obj) _PyStackRef_FromPyObjectStealMortal(_PyObject_CAST(obj), __FILE__, __LINE__)

static inline bool
PyStackRef_IsHeapSafe(_PyStackRef ref)
{
    if ((ref.index & Py_TAG_BITS) != Py_TAG_REFCNT || PyStackRef_IsNull(ref)) {
        // Tagged ints and ERROR are included.
        return true;
    }

    PyObject *obj = _Py_stackref_get_object(ref);
    return _Py_IsImmortal(obj);
}

static inline _PyStackRef
_PyStackRef_MakeHeapSafe(_PyStackRef ref, const char *filename, int linenumber)
{
    // Special references that can't be closed.
    if (ref.index < INITIAL_STACKREF_INDEX) {
        return ref;
    }

    bool heap_safe = PyStackRef_IsHeapSafe(ref);
    PyObject *obj = _Py_stackref_close(ref, filename, linenumber);
    uint16_t flags = 0;
    if (heap_safe) {
        // Close old ref and create a new one with the same flags.
        // This is necessary for correct borrow checking.
        flags = ref.index & Py_TAG_BITS;
    } else {
        Py_INCREF(obj);
    }
    return _Py_stackref_create(obj, flags, filename, linenumber);
}
#define PyStackRef_MakeHeapSafe(REF) _PyStackRef_MakeHeapSafe(REF, __FILE__, __LINE__)

static inline _PyStackRef
_PyStackRef_FromPyObjectNewMortal(PyObject *obj, const char *filename, int linenumber)
{
    assert(obj != NULL);
    assert(!_Py_IsStaticImmortal(obj));
    Py_INCREF(obj);
    return _Py_stackref_create(obj, 0, filename, linenumber);
}
#define PyStackRef_FromPyObjectNewMortal(obj) _PyStackRef_FromPyObjectNewMortal(_PyObject_CAST(obj), __FILE__, __LINE__)

extern int PyStackRef_Is(_PyStackRef a, _PyStackRef b);

extern bool PyStackRef_IsTaggedInt(_PyStackRef ref);

extern intptr_t PyStackRef_UntagInt(_PyStackRef ref);

extern _PyStackRef PyStackRef_TagInt(intptr_t i);

/* Increments a tagged int, but does not check for overflow */
extern _PyStackRef PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref);

extern bool
PyStackRef_IsNullOrInt(_PyStackRef ref);

#else

static const _PyStackRef PyStackRef_ERROR = { .bits = Py_TAG_INVALID };

/* For use in the JIT to clear an unused value.
 * PyStackRef_ZERO_BITS has no meaning and should not be used other than by the JIT. */
static const _PyStackRef PyStackRef_ZERO_BITS = { .bits = 0 };

/* Wrap a pointer in a stack ref.
 * The resulting stack reference is not safe and should only be used
 * in the interpreter to pass values from one uop to another.
 * The GC should never see one of these stack refs. */
static inline _PyStackRef
PyStackRef_Wrap(void *ptr)
{
    assert(ptr != NULL);
#ifdef Py_DEBUG
    assert(((uintptr_t)ptr & Py_TAG_BITS) == 0);
    return (_PyStackRef){ .bits = ((uintptr_t)ptr) | Py_TAG_INVALID };
#else
    return (_PyStackRef){ .bits = (uintptr_t)ptr };
#endif
}

static inline void *
PyStackRef_Unwrap(_PyStackRef ref)
{
#ifdef Py_DEBUG
    assert ((ref.bits & Py_TAG_BITS) == Py_TAG_INVALID);
    return (void *)(ref.bits & ~Py_TAG_BITS);
#else
    return (void *)(ref.bits);
#endif
}

static inline bool
PyStackRef_IsError(_PyStackRef ref)
{
    return ref.bits == Py_TAG_INVALID;
}

static inline bool
PyStackRef_IsMalformed(_PyStackRef ref)
{
    return (ref.bits & Py_TAG_BITS) == Py_TAG_INVALID;
}

static inline bool
PyStackRef_IsValid(_PyStackRef ref)
{
    /* Invalid values are ERROR and NULL */
    return ref.bits >= Py_INT_TAG;
}

static inline bool
PyStackRef_IsTaggedInt(_PyStackRef i)
{
    return (i.bits & Py_TAG_BITS) == Py_INT_TAG;
}

static inline _PyStackRef
PyStackRef_TagInt(intptr_t i)
{
    assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (intptr_t)(((uintptr_t)i) << Py_TAGGED_SHIFT),
                                     Py_TAGGED_SHIFT) == i);
    return (_PyStackRef){ .bits = ((((uintptr_t)i) << Py_TAGGED_SHIFT) | Py_INT_TAG) };
}

static inline intptr_t
PyStackRef_UntagInt(_PyStackRef i)
{
    assert(PyStackRef_IsTaggedInt(i));
    intptr_t val = (intptr_t)i.bits;
    return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, Py_TAGGED_SHIFT);
}


static inline _PyStackRef
PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref)
{
    assert((ref.bits & Py_TAG_BITS) == Py_INT_TAG); // Is tagged int
    assert((ref.bits & (~Py_TAG_BITS)) != (INTPTR_MAX & (~Py_TAG_BITS))); // Isn't about to overflow
    return (_PyStackRef){ .bits = ref.bits + (1 << Py_TAGGED_SHIFT) };
}

/* References to immortal objects always have their tag bit set to Py_TAG_REFCNT
 * as they can (must) have their reclamation deferred */

#if _Py_IMMORTAL_FLAGS != Py_TAG_REFCNT
#  error "_Py_IMMORTAL_FLAGS != Py_TAG_REFCNT"
#endif

#define BITS_TO_PTR(REF) ((PyObject *)((REF).bits))
#define BITS_TO_PTR_MASKED(REF) ((PyObject *)(((REF).bits) & (~Py_TAG_REFCNT)))

#define PyStackRef_NULL_BITS Py_TAG_REFCNT
static const _PyStackRef PyStackRef_NULL = { .bits = PyStackRef_NULL_BITS };

#define PyStackRef_IsNull(ref) ((ref).bits == PyStackRef_NULL_BITS)
#define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_REFCNT })
#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT })
#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT })

#ifdef Py_GIL_DISABLED
// Checks that mask out the deferred bit in the free threading build.
#define PyStackRef_IsNone(REF) (((REF).bits & ~Py_TAG_REFCNT) == (uintptr_t)&_Py_NoneStruct)
#define PyStackRef_IsTrue(REF) (((REF).bits & ~Py_TAG_REFCNT) == (uintptr_t)&_Py_TrueStruct)
#define PyStackRef_IsFalse(REF) (((REF).bits & ~Py_TAG_REFCNT) == (uintptr_t)&_Py_FalseStruct)
#else
#define PyStackRef_IsTrue(REF) ((REF).bits == (((uintptr_t)&_Py_TrueStruct) | Py_TAG_REFCNT))
#define PyStackRef_IsFalse(REF) ((REF).bits == (((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT))
#define PyStackRef_IsNone(REF) ((REF).bits == (((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT))
#endif

#define PyStackRef_IsNullOrInt(stackref) (PyStackRef_IsNull(stackref) || PyStackRef_IsTaggedInt(stackref))

#if defined(Py_DEBUG) && !defined(Py_GIL_DISABLED)

static inline void
PyStackRef_CheckValid(_PyStackRef ref)
{
    assert(ref.bits != 0);
    int tag = ref.bits & Py_TAG_BITS;
    PyObject *obj = BITS_TO_PTR_MASKED(ref);
    switch (tag) {
        case 0:
            /* Can be immortal if object was made immortal after reference came into existence */
            assert(!_Py_IsStaticImmortal(obj));
            break;
        case Py_TAG_REFCNT:
            break;
        default:
            assert(0);
    }
}

#else

#define PyStackRef_CheckValid(REF) ((void)0)

#endif

#ifdef _WIN32
#define PyStackRef_RefcountOnObject(REF) (((REF).bits & Py_TAG_REFCNT) == 0)
#define PyStackRef_AsPyObjectBorrow BITS_TO_PTR_MASKED
#define PyStackRef_Borrow(REF) (_PyStackRef){ .bits = ((REF).bits) | Py_TAG_REFCNT};
#else
/* Does this ref not have an embedded refcount and thus not refer to a declared immmortal object? */
static inline int
PyStackRef_RefcountOnObject(_PyStackRef ref)
{
    return (ref.bits & Py_TAG_REFCNT) == 0;
}

static inline PyObject *
PyStackRef_AsPyObjectBorrow(_PyStackRef ref)
{
    assert(!PyStackRef_IsTaggedInt(ref));
    return BITS_TO_PTR_MASKED(ref);
}

static inline _PyStackRef
PyStackRef_Borrow(_PyStackRef ref)
{
    return (_PyStackRef){ .bits = ref.bits | Py_TAG_REFCNT };
}
#endif

static inline PyObject *
PyStackRef_AsPyObjectSteal(_PyStackRef ref)
{
    assert(!PyStackRef_IsNull(ref));
    assert(!PyStackRef_IsTaggedInt(ref));
    if (PyStackRef_RefcountOnObject(ref)) {
        return BITS_TO_PTR(ref);
    }
    else {
        return Py_NewRef(BITS_TO_PTR_MASKED(ref));
    }
}

static inline _PyStackRef
PyStackRef_FromPyObjectSteal(PyObject *obj)
{
    assert(obj != NULL);
#ifdef Py_GIL_DISABLED
    return (_PyStackRef){ .bits = (uintptr_t)obj };
#else
#  if SIZEOF_VOID_P > 4
    unsigned int tag = obj->ob_flags & Py_TAG_REFCNT;
#  else
    unsigned int tag = _Py_IsImmortal(obj) ? Py_TAG_REFCNT : 0;
#  endif
    _PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag});
    PyStackRef_CheckValid(ref);
    return ref;
#endif
}

static inline _PyStackRef
PyStackRef_FromPyObjectStealMortal(PyObject *obj)
{
    assert(obj != NULL);
    assert(!_Py_IsImmortal(obj));
    _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj };
    PyStackRef_CheckValid(ref);
    return ref;
}

static inline _PyStackRef
_PyStackRef_FromPyObjectNew(PyObject *obj)
{
    assert(obj != NULL);
#ifdef Py_GIL_DISABLED
    if (_PyObject_HasDeferredRefcount(obj)) {
        return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT };
    }
#else
    if (_Py_IsImmortal(obj)) {
        return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT };
    }
#endif
    _Py_INCREF_MORTAL(obj);
    _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj };
    PyStackRef_CheckValid(ref);
    return ref;
}
#define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj))

static inline _PyStackRef
_PyStackRef_FromPyObjectNewMortal(PyObject *obj)
{
    assert(obj != NULL);
    _Py_INCREF_MORTAL(obj);
    _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj };
    PyStackRef_CheckValid(ref);
    return ref;
}
#define PyStackRef_FromPyObjectNewMortal(obj) _PyStackRef_FromPyObjectNewMortal(_PyObject_CAST(obj))

/* Create a new reference from an object with an embedded reference count */
static inline _PyStackRef
PyStackRef_FromPyObjectBorrow(PyObject *obj)
{
    assert(obj != NULL);
    return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT};
}

/* Tag a PyObject pointer as a borrowed operand for BORROW variants. */
static inline uintptr_t
PyStackRef_TagBorrow(PyObject *obj)
{
    return (uintptr_t)obj | Py_TAG_REFCNT;
}

/* Strip tag bits from a pre-tagged operand to recover the PyObject pointer. */
static inline PyObject *
PyStackRef_UntagBorrow(PyObject *tagged)
{
    return (PyObject *)((uintptr_t)tagged & ~Py_TAG_BITS);
}

/* Create a stackref from a pre-tagged operand (tag bits already set).
   Used by _LOAD_CONST_INLINE_BORROW variants where the operand is
   tagged at trace creation time to avoid tagging on every execution. */
static inline _PyStackRef
PyStackRef_FromPreTagged(PyObject *tagged)
{
    assert((uintptr_t)tagged & Py_TAG_REFCNT);
    return (_PyStackRef){ .bits = (uintptr_t)tagged };
}

/* WARNING: This macro evaluates its argument more than once */
#ifdef _WIN32
#define PyStackRef_DUP(REF) \
    (PyStackRef_RefcountOnObject(REF) ? (_Py_INCREF_MORTAL(BITS_TO_PTR(REF)), (REF)) : (REF))
#else
static inline _PyStackRef
PyStackRef_DUP(_PyStackRef ref)
{
    assert(!PyStackRef_IsNull(ref));
    if (PyStackRef_RefcountOnObject(ref)) {
        _Py_INCREF_MORTAL(BITS_TO_PTR(ref));
    }
    return ref;
}
#endif

static inline bool
PyStackRef_IsHeapSafe(_PyStackRef ref)
{
#ifdef Py_GIL_DISABLED
    if ((ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT) {
        return true;
    }
    PyObject *obj = BITS_TO_PTR_MASKED(ref);
    return obj == NULL || _PyObject_HasDeferredRefcount(obj);
#else
    return (ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref));
#endif
}

static inline _PyStackRef
PyStackRef_MakeHeapSafe(_PyStackRef ref)
{
    if (PyStackRef_IsHeapSafe(ref)) {
        return ref;
    }
    PyObject *obj = BITS_TO_PTR_MASKED(ref);
    Py_INCREF(obj);
    ref.bits = (uintptr_t)obj;
    PyStackRef_CheckValid(ref);
    return ref;
}

// Convert a possibly deferred reference to a strong reference.
static inline _PyStackRef
PyStackRef_AsStrongReference(_PyStackRef stackref)
{
    return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref));
}

#ifdef _WIN32
#define PyStackRef_CLOSE(REF) \
do { \
    _PyStackRef _temp = (REF); \
    if (PyStackRef_RefcountOnObject(_temp)) Py_DECREF_MORTAL(BITS_TO_PTR(_temp)); \
} while (0)
#else
static inline void
PyStackRef_CLOSE(_PyStackRef ref)
{
    assert(!PyStackRef_IsNull(ref));
    if (PyStackRef_RefcountOnObject(ref)) {
        Py_DECREF_MORTAL(BITS_TO_PTR(ref));
    }
}
#endif

static inline void
PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
{
    assert(!PyStackRef_IsNull(ref));
    if (PyStackRef_RefcountOnObject(ref)) {
        Py_DECREF_MORTAL_SPECIALIZED(BITS_TO_PTR(ref), destruct);
    }
}

#ifdef _WIN32
#define PyStackRef_XCLOSE PyStackRef_CLOSE
#else
static inline void
PyStackRef_XCLOSE(_PyStackRef ref)
{
    assert(ref.bits != 0);
    if (PyStackRef_RefcountOnObject(ref)) {
        assert(!PyStackRef_IsNull(ref));
        Py_DECREF_MORTAL(BITS_TO_PTR(ref));
    }
}
#endif

#define PyStackRef_CLEAR(REF) \
    do { \
        _PyStackRef *_tmp_op_ptr = &(REF); \
        _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \
        *_tmp_op_ptr = PyStackRef_NULL; \
        PyStackRef_XCLOSE(_tmp_old_op); \
    } while (0)


// Note: this is a macro because MSVC (Windows) has trouble inlining it.

#define PyStackRef_Is(a, b) (((a).bits & (~Py_TAG_REFCNT)) == ((b).bits & (~Py_TAG_REFCNT)))


#endif // !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)

static inline PyTypeObject *
PyStackRef_TYPE(_PyStackRef stackref) {
    if (PyStackRef_IsTaggedInt(stackref)) {
        return &PyLong_Type;
    }
    return Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref));
}

// Converts a PyStackRef back to a PyObject *, converting the
// stackref to a new reference.
#define PyStackRef_AsPyObjectNew(stackref) Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref))

// StackRef type checks

#define STACKREF_CHECK_FUNC(T) \
    static inline bool \
    PyStackRef_ ## T ## Check(_PyStackRef stackref) { \
        if (PyStackRef_IsTaggedInt(stackref)) { \
            return false; \
        } \
        return Py ## T ## _Check(PyStackRef_AsPyObjectBorrow(stackref)); \
    }

STACKREF_CHECK_FUNC(Gen)
STACKREF_CHECK_FUNC(Bool)
STACKREF_CHECK_FUNC(ExceptionInstance)
STACKREF_CHECK_FUNC(Code)
STACKREF_CHECK_FUNC(Function)

static inline bool
PyStackRef_LongCheck(_PyStackRef stackref)
{
    if (PyStackRef_IsTaggedInt(stackref)) {
        return true;
    }
    return PyLong_Check(PyStackRef_AsPyObjectBorrow(stackref));
}

static inline void
_PyThreadState_PushCStackRef(PyThreadState *tstate, _PyCStackRef *ref)
{
#ifdef Py_GIL_DISABLED
    _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
    ref->next = tstate_impl->c_stack_refs;
    tstate_impl->c_stack_refs = ref;
#endif
    ref->ref = PyStackRef_NULL;
}

static inline void
_PyThreadState_PushCStackRefNew(PyThreadState *tstate, _PyCStackRef *ref, PyObject *obj)
{
    _PyThreadState_PushCStackRef(tstate, ref);
    ref->ref = PyStackRef_FromPyObjectNew(obj);
}

static inline void
_PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref)
{
#ifdef Py_GIL_DISABLED
    _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
    assert(tstate_impl->c_stack_refs == ref);
    tstate_impl->c_stack_refs = ref->next;
#endif
    PyStackRef_XCLOSE(ref->ref);
}

static inline _PyStackRef
_PyThreadState_PopCStackRefSteal(PyThreadState *tstate, _PyCStackRef *ref)
{
#ifdef Py_GIL_DISABLED
    _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
    assert(tstate_impl->c_stack_refs == ref);
    tstate_impl->c_stack_refs = ref->next;
#endif
    return ref->ref;
}

#ifdef Py_GIL_DISABLED

static inline int
_Py_TryIncrefCompareStackRef(PyObject **src, PyObject *op, _PyStackRef *out)
{
    if (_PyObject_HasDeferredRefcount(op)) {
        *out = (_PyStackRef){ .bits = (uintptr_t)op | Py_TAG_REFCNT };
        return 1;
    }
    if (_Py_TryIncrefCompare(src, op)) {
        *out = PyStackRef_FromPyObjectSteal(op);
        return 1;
    }
    return 0;
}

static inline int
_Py_TryXGetStackRef(PyObject **src, _PyStackRef *out)
{
    PyObject *op = _PyObject_CAST(_Py_atomic_load_ptr_relaxed(src));
    if (op == NULL) {
        *out = PyStackRef_NULL;
        return 1;
    }
    return _Py_TryIncrefCompareStackRef(src, op, out);
}

#endif

#define PyStackRef_XSETREF(dst, src) \
    do { \
        _PyStackRef _tmp_dst_ref = (dst); \
        (dst) = (src); \
        PyStackRef_XCLOSE(_tmp_dst_ref); \
    } while(0)

// Like Py_VISIT but for _PyStackRef fields
#define _Py_VISIT_STACKREF(ref)                                         \
    do {                                                                \
        if (!PyStackRef_IsNullOrInt(ref)) {                             \
            int vret = _PyGC_VisitStackRef(&(ref), visit, arg);         \
            if (vret)                                                   \
                return vret;                                            \
        }                                                               \
    } while (0)

static inline void
_PyStackRef_CloseStack(_PyStackRef *arguments, int total_args)
{
    // arguments is a pointer into the GC visible stack,
    // so we must NULL out values as we clear them.
    for (int i = total_args-1; i >= 0; i--) {
        _PyStackRef tmp = arguments[i];
        arguments[i] = PyStackRef_NULL;
        PyStackRef_CLOSE(tmp);
    }
}

#ifdef __cplusplus
}
#endif
#endif /* !Py_INTERNAL_STACKREF_H */