From 2264fe8b30cfde7f59ebda0b184576eb7a0a0532 Mon Sep 17 00:00:00 2001 From: Binyamin Y Cohen <42520501+binyamin555@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:58:06 +0200 Subject: [PATCH] Add Python 3.14 and free-threaded (3.14t) support - Build and test on Python 3.14 and 3.14t in CI and the publish workflow - Declare GIL-free support and serialize type mutations on free-threaded builds (guarded by Py_GIL_DISABLED) - Fix reference leaks in all slot implementations - Fix __getitem__ not receiving the key, __setitem__/__delitem__ failing, __index__ and Py_ssize_t sequence slots crashing, __iadd__ using the wrong sequence slot, __hash__ returning -1 and __bool__ error handling - Invalidate the type attribute cache when extending a type, which could crash after the replaced method was freed - Remove `del _T`, which breaks lazily evaluated annotations on 3.14 - Run the Windows publish job on windows-latest (windows-2019 is retired) Co-Authored-By: Claude Opus 5.5 --- .github/workflows/python-publish.yml | 9 +- .github/workflows/python-test-linux.yml | 2 +- .github/workflows/python-test-macos.yml | 2 +- .github/workflows/python-test-windows.yml | 2 +- README.md | 6 + src/extype/core.pyi | 3 + src/extype/extension_utils.py | 10 +- src/extype/extype.cpp | 425 ++++++++++++++++------ tests/test_free_threading.py | 88 +++++ tests/test_slot_implementations.py | 186 ++++++++++ 10 files changed, 615 insertions(+), 118 deletions(-) create mode 100644 tests/test_free_threading.py create mode 100644 tests/test_slot_implementations.py diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 2bebbb6..0a0c303 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v3 @@ -38,6 +38,7 @@ jobs: # if: github.event.inputs.build == 'true' # run: python -m build --wheel --sdist --outdir dist/ env: + # Also builds the free-threaded (cp3XXt) wheels for versions that support them CIBW_PROJECT_REQUIRES_PYTHON: ==${{ matrix.python-version }}.* run: python -m cibuildwheel --output-dir dist/ - name: Upload Artifact @@ -49,10 +50,10 @@ jobs: windows: name: Build for Windows - runs-on: windows-2019 + runs-on: windows-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] steps: - uses: actions/checkout@v3 @@ -82,7 +83,7 @@ jobs: runs-on: macos-15-intel strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/python-test-linux.yml b/.github/workflows/python-test-linux.yml index 9bd5f81..3a3526e 100644 --- a/.github/workflows/python-test-linux.yml +++ b/.github/workflows/python-test-linux.yml @@ -50,7 +50,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/python-test-macos.yml b/.github/workflows/python-test-macos.yml index 96208d6..a0083e8 100644 --- a/.github/workflows/python-test-macos.yml +++ b/.github/workflows/python-test-macos.yml @@ -50,7 +50,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/python-test-windows.yml b/.github/workflows/python-test-windows.yml index d7d7c79..cbb0855 100644 --- a/.github/workflows/python-test-windows.yml +++ b/.github/workflows/python-test-windows.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] steps: - uses: actions/checkout@v3 diff --git a/README.md b/README.md index 9367179..1fdc22a 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,12 @@ Only methods marked with `@extension` will be added as extension methods. Extending a type will extend it in all modules, not just the one that called the `extend_type_with`, so make sure you don't override an existing function, unless, of course, it is what you want. +### Free-threaded Python +ExType supports the free-threaded build of Python (3.14t and later) and does not re-enable the GIL when imported. +Extending a type is thread-safe, but other threads that are using the type at the same time may see either +the old or the new behavior until `extend_type_with` returns. It is recommended to apply your extensions +at import time, before starting other threads. + ## Features - [x] Exteranlly extend type via another type diff --git a/src/extype/core.pyi b/src/extype/core.pyi index 4f8804d..800d8b7 100644 --- a/src/extype/core.pyi +++ b/src/extype/core.pyi @@ -6,6 +6,9 @@ if TYPE_CHECKING: def get_type_dict(typ: Type) -> Dict[str, Any]: ... +def set_type_attribute(typ: Type, name: str, value: Any) -> None: ... + + def implement_protocol_on_type(typ: Type, protocols: "Protocol"): ... diff --git a/src/extype/extension_utils.py b/src/extype/extension_utils.py index 639d581..a02e32e 100644 --- a/src/extype/extension_utils.py +++ b/src/extype/extension_utils.py @@ -1,6 +1,6 @@ from enum import IntFlag from typing import Generic, TypeVar -from .core import enable_magic_method, get_type_dict +from .core import enable_magic_method, set_type_attribute _T = TypeVar("_T") @@ -24,8 +24,6 @@ def extend_type_with(typ, cls, recurse_parents=True): if not isinstance(typ, type): raise TypeError(f"argument 'typ' must be a valid type, got {type(typ)}") - type_dict = get_type_dict(typ) - if recurse_parents: for base in cls.__bases__: extend_type_with(typ, base) @@ -36,13 +34,15 @@ def extend_type_with(typ, cls, recurse_parents=True): item = item.original if callable(getattr(item, "__set_name__", None)): item.__set_name__(typ, name) + # Set the attribute before enabling the slot, so the slot + # never runs without a method to call. + set_type_attribute(typ, name, item) if callable(item): if name.startswith("__") and name.endswith("__"): try: enable_magic_method(typ, name) except ValueError: ... - type_dict[name] = item class ExtensionObject(Generic[_T]): @@ -59,8 +59,6 @@ def extension(o: _T) -> _T: raise TypeError("Only functions and descriptors may be marked as extensions") -del _T - __all__ = [ "extension", "extend_type_with" diff --git a/src/extype/extype.cpp b/src/extype/extype.cpp index 671ccde..1187736 100644 --- a/src/extype/extype.cpp +++ b/src/extype/extype.cpp @@ -46,71 +46,145 @@ struct MagicMethodLookupResult static constexpr uint32_t INVALID_MAGIC_METHOD = 1U; static constexpr uint32_t NOT_IMPLEMENTED = 2U; +// Free-threading support +// +// On free-threaded builds, changes to type slots are serialized with a mutex, and slot +// pointers are published atomically since other threads read them without any lock. +// On regular builds the GIL already provides both guarantees, so the macros do nothing. + +#ifdef Py_GIL_DISABLED + +static PyMutex type_mutation_mutex = {0}; + +#define TYPE_MUTATION_LOCK() PyMutex_Lock(&type_mutation_mutex) +#define TYPE_MUTATION_UNLOCK() PyMutex_Unlock(&type_mutation_mutex) +#define STORE_SLOT(slot, value) _Py_atomic_store_ptr_release((slot), (value)) + +#else + +#define TYPE_MUTATION_LOCK() +#define TYPE_MUTATION_UNLOCK() +#define STORE_SLOT(slot, value) (*(slot) = (value)) + +#endif + +// Helpers + +// Calls `self.(*args)` and returns a new reference to the result. +// Steals the reference to `args`, which may be NULL if building it failed. +static PyObject *call_method(PyObject *self, char const *name, PyObject *args) +{ + if (args == NULL) + { + return NULL; + } + + PyObject *method = PyObject_GetAttrString(self, name); + if (method == NULL) + { + Py_DECREF(args); + return NULL; + } + + PyObject *result = PyObject_Call(method, args, NULL); + Py_DECREF(method); + Py_DECREF(args); + return result; +} + +// Like `call_method`, for slots that return 0 on success and -1 on error. +static int call_method_status(PyObject *self, char const *name, PyObject *args) +{ + PyObject *result = call_method(self, name, args); + if (result == NULL) + { + return -1; + } + Py_DECREF(result); + return 0; +} + +// Like `call_method`, for slots that return the truth value of the result. +static int call_method_truth(PyObject *self, char const *name, PyObject *args) +{ + PyObject *result = call_method(self, name, args); + if (result == NULL) + { + return -1; + } + int truth = PyObject_IsTrue(result); + Py_DECREF(result); + return truth; +} + +// Like `call_method`, for slots that return the result as a Py_ssize_t. +static Py_ssize_t call_method_ssize(PyObject *self, char const *name, PyObject *args) +{ + PyObject *result = call_method(self, name, args); + if (result == NULL) + { + return -1; + } + Py_ssize_t value = PyNumber_AsSsize_t(result, NULL); + Py_DECREF(result); + return value; +} + // Implementations -#define UNARY_OP_IMPL(name) \ - static PyObject *impl##name(PyObject *self) \ - { \ - if (!PyObject_HasAttrString(self, #name)) \ - { \ - Py_RETURN_NOTIMPLEMENTED; \ - } \ - return PyObject_Call(PyObject_GetAttrString(self, #name), PyTuple_Pack(0), NULL); \ +#define UNARY_OP_IMPL(name) \ + static PyObject *impl##name(PyObject *self) \ + { \ + if (!PyObject_HasAttrString(self, #name)) \ + { \ + Py_RETURN_NOTIMPLEMENTED; \ + } \ + return call_method(self, #name, PyTuple_Pack(0)); \ } #define UNARY_OP(name) UNARY_OP_IMPL(__##name##__) -#define BINARY_OP_IMPL(name) \ - static PyObject *impl##name(PyObject *left, PyObject *right) \ - { \ - if (!PyObject_HasAttrString(left, #name)) \ - { \ - Py_RETURN_NOTIMPLEMENTED; \ - } \ - return PyObject_Call(PyObject_GetAttrString(left, #name), PyTuple_Pack(1, right), NULL); \ +#define BINARY_OP_IMPL(name) \ + static PyObject *impl##name(PyObject *left, PyObject *right) \ + { \ + if (!PyObject_HasAttrString(left, #name)) \ + { \ + Py_RETURN_NOTIMPLEMENTED; \ + } \ + return call_method(left, #name, PyTuple_Pack(1, right)); \ } #define BINARY_OP(name) BINARY_OP_IMPL(__##name##__) -#define _RBINARY_OP_IMPL(name, rname) \ - static PyObject *impl##name(PyObject *left, PyObject *right) \ - { \ - if (!PyObject_HasAttrString(left, #name)) \ - { \ - if (!PyObject_HasAttrString(right, #rname)) \ - { \ - Py_RETURN_NOTIMPLEMENTED; \ - } \ - else \ - { \ - return PyObject_Call(PyObject_GetAttrString(right, #rname), PyTuple_Pack(1, left), NULL); \ - } \ - } \ - PyObject *result = PyObject_Call(PyObject_GetAttrString(left, #name), PyTuple_Pack(1, right), NULL); \ - if (result == Py_NotImplemented) \ - { \ - if (!PyObject_HasAttrString(right, #rname)) \ - { \ - Py_RETURN_NOTIMPLEMENTED; \ - } \ - else \ - { \ - return PyObject_Call(PyObject_GetAttrString(right, #rname), PyTuple_Pack(1, left), NULL); \ - } \ - } \ - return result; \ +#define _RBINARY_OP_IMPL(name, rname) \ + static PyObject *impl##name(PyObject *left, PyObject *right) \ + { \ + if (PyObject_HasAttrString(left, #name)) \ + { \ + PyObject *result = call_method(left, #name, PyTuple_Pack(1, right)); \ + if (result != Py_NotImplemented) \ + { \ + return result; \ + } \ + Py_DECREF(result); \ + } \ + if (!PyObject_HasAttrString(right, #rname)) \ + { \ + Py_RETURN_NOTIMPLEMENTED; \ + } \ + return call_method(right, #rname, PyTuple_Pack(1, left)); \ } #define RBINARY_OP(name) _RBINARY_OP_IMPL(__##name##__, __r##name##__) -#define TERNARY_OP_IMPL(name) \ - static PyObject *impl##name(PyObject *self, PyObject *left, PyObject *right) \ - { \ - if (!PyObject_HasAttrString(self, #name)) \ - { \ - Py_RETURN_NOTIMPLEMENTED; \ - } \ - return PyObject_Call(PyObject_GetAttrString(self, #name), PyTuple_Pack(2, left, right), NULL); \ +#define TERNARY_OP_IMPL(name) \ + static PyObject *impl##name(PyObject *self, PyObject *left, PyObject *right) \ + { \ + if (!PyObject_HasAttrString(self, #name)) \ + { \ + Py_RETURN_NOTIMPLEMENTED; \ + } \ + return call_method(self, #name, PyTuple_Pack(2, left, right)); \ } #define TERNARY_OP(name) TERNARY_OP_IMPL(__##name##__) @@ -135,11 +209,12 @@ static int impl__bool__(PyObject *self) PyErr_SetString( PyExc_NotImplementedError, (std::string{"Type "} + - _PyType_Name((PyTypeObject *)PyObject_Type(self)) + + _PyType_Name(Py_TYPE(self)) + " does not implement the '__bool__' method") .c_str()); + return -1; } - return PyObject_IsTrue(PyObject_Call(PyObject_GetAttrString(self, "__bool__"), PyTuple_Pack(0), NULL)); + return call_method_truth(self, "__bool__", PyTuple_Pack(0)); } UNARY_OP(invert) @@ -167,7 +242,7 @@ RBINARY_OP(truediv) BINARY_OP(ifloordiv) BINARY_OP(itruediv) -BINARY_OP(index) +UNARY_OP(index) RBINARY_OP(matmul) BINARY_OP(imatmul) @@ -176,48 +251,87 @@ BINARY_OP(imatmul) static Py_ssize_t impl__len__(PyObject *self) { - return PyNumber_AsSsize_t(PyObject_Call(PyObject_GetAttrString(self, "__len__"), PyTuple_Pack(0), NULL), NULL); + return call_method_ssize(self, "__len__", PyTuple_Pack(0)); } static int impl__contains__(PyObject *self, PyObject *value) { - return PyNumber_AsSsize_t(PyObject_Call(PyObject_GetAttrString(self, "__contains__"), PyTuple_Pack(1, value), NULL), NULL); + return call_method_truth(self, "__contains__", PyTuple_Pack(1, value)); } static PyObject *impl__getitem__(PyObject *self, PyObject *item) { - return PyObject_Call(PyObject_GetAttrString(self, "__getitem__"), PyTuple_Pack(0), NULL); + return call_method(self, "__getitem__", PyTuple_Pack(1, item)); } static int impl__setitem__(PyObject *self, PyObject *item, PyObject *value) { if (value == NULL) { - return PyNumber_AsSsize_t(PyObject_Call(PyObject_GetAttrString(self, "__delitem__"), PyTuple_Pack(1, item), NULL), NULL); + return call_method_status(self, "__delitem__", PyTuple_Pack(1, item)); } - return PyNumber_AsSsize_t(PyObject_Call(PyObject_GetAttrString(self, "__setitem__"), PyTuple_Pack(2, item, value), NULL), NULL); + return call_method_status(self, "__setitem__", PyTuple_Pack(2, item, value)); +} + +// Sequence slots that take a Py_ssize_t instead of an object + +static PyObject *impl_sq_item(PyObject *self, Py_ssize_t index) +{ + return call_method(self, "__getitem__", Py_BuildValue("(n)", index)); +} + +static int impl_sq_ass_item(PyObject *self, Py_ssize_t index, PyObject *value) +{ + if (value == NULL) + { + return call_method_status(self, "__delitem__", Py_BuildValue("(n)", index)); + } + return call_method_status(self, "__setitem__", Py_BuildValue("(nO)", index, value)); +} + +static PyObject *impl_sq_repeat(PyObject *self, Py_ssize_t count) +{ + return call_method(self, "__mul__", Py_BuildValue("(n)", count)); +} + +static PyObject *impl_sq_inplace_repeat(PyObject *self, Py_ssize_t count) +{ + return call_method(self, "__imul__", Py_BuildValue("(n)", count)); } // Special methods static PyObject *impl__repr__(PyObject *self) { - return PyObject_Call(PyObject_GetAttrString(self, "__repr__"), PyTuple_Pack(0), NULL); + return call_method(self, "__repr__", PyTuple_Pack(0)); } static PyObject *impl__call__(PyObject *self, PyObject *args, PyObject *kwargs) { - return PyObject_Call(PyObject_GetAttrString(self, "__call__"), args, kwargs); + PyObject *method = PyObject_GetAttrString(self, "__call__"); + if (method == NULL) + { + return NULL; + } + PyObject *result = PyObject_Call(method, args, kwargs); + Py_DECREF(method); + return result; } static Py_hash_t impl__hash__(PyObject *self) { - return PyNumber_AsSsize_t(PyObject_Call(PyObject_GetAttrString(self, "__hash__"), PyTuple_Pack(0), NULL), NULL); + Py_hash_t hash = call_method_ssize(self, "__hash__", PyTuple_Pack(0)); + // -1 is reserved for errors, so a successful hash of -1 becomes -2 (like Python does) + if (hash == -1 && !PyErr_Occurred()) + { + hash = -2; + } + return hash; } static PyObject *impl__str__(PyObject *self) { - return PyObject_Call(PyObject_GetAttrString(self, "__str__"), PyTuple_Pack(0), NULL); + return call_method(self, "__str__", PyTuple_Pack(0)); } // PyObject_GenericGetAttr @@ -232,12 +346,12 @@ static PyObject *impl__str__(PyObject *self) static PyObject *impl__iter__(PyObject *self) { - return PyObject_Call(PyObject_GetAttrString(self, "__iter__"), PyTuple_Pack(0), NULL); + return call_method(self, "__iter__", PyTuple_Pack(0)); } static PyObject *impl__next__(PyObject *self) { - return PyObject_Call(PyObject_GetAttrString(self, "__next__"), PyTuple_Pack(0), NULL); + return call_method(self, "__next__", PyTuple_Pack(0)); } // End special methods @@ -249,52 +363,93 @@ static PyObject *implement_protocol_on_type(PyObject *self, PyObject *args) PyTypeObject *type; uint8_t protocols; - PyArg_ParseTuple(args, "OB", &type, &protocols); + if (!PyArg_ParseTuple(args, "O!B", &PyType_Type, &type, &protocols)) + { + return NULL; + } + + TYPE_MUTATION_LOCK(); if (protocols & Protocols::Number && type->tp_as_number == NULL) { - PyNumberMethods *protocol = new PyNumberMethods(); - type->tp_as_number = protocol; + STORE_SLOT(&type->tp_as_number, new PyNumberMethods()); } if (protocols & Protocols::Sequence && type->tp_as_sequence == NULL) { - PySequenceMethods *protocol = new PySequenceMethods(); - type->tp_as_sequence = protocol; + STORE_SLOT(&type->tp_as_sequence, new PySequenceMethods()); } if (protocols & Protocols::Mapping && type->tp_as_mapping == NULL) { - PyMappingMethods *protocol = new PyMappingMethods(); - type->tp_as_mapping = protocol; + STORE_SLOT(&type->tp_as_mapping, new PyMappingMethods()); } + TYPE_MUTATION_UNLOCK(); + Py_RETURN_NONE; } -static PyObject *get_type_dict(PyObject *self, PyObject *args) +// Returns a new reference to the type's dict +static PyObject *type_dict_of(PyTypeObject *type) { - PyTypeObject *type; - PyArg_ParseTuple(args, "O", &type); - #if PY_VERSION_HEX >= 0x030C0000 - auto dict = PyType_GetDict(type); + PyObject *dict = PyType_GetDict(type); #else - - auto dict = type->tp_dict; - + + PyObject *dict = type->tp_dict; + Py_XINCREF(dict); + #endif if (dict == NULL) { PyErr_SetString(PyExc_RuntimeError, "type dict was null"); - return NULL; } - - Py_INCREF(dict); return dict; } +static PyObject *get_type_dict(PyObject *self, PyObject *args) +{ + PyTypeObject *type; + if (!PyArg_ParseTuple(args, "O!", &PyType_Type, &type)) + { + return NULL; + } + + return type_dict_of(type); +} + +static PyObject *set_type_attribute(PyObject *self, PyObject *args) +{ + PyTypeObject *type; + PyObject *name; + PyObject *value; + if (!PyArg_ParseTuple(args, "O!UO", &PyType_Type, &type, &name, &value)) + { + return NULL; + } + + PyObject *dict = type_dict_of(type); + if (dict == NULL) + { + return NULL; + } + + int status = PyDict_SetItem(dict, name, value); + Py_DECREF(dict); + if (status < 0) + { + return NULL; + } + + // Writing to the dict directly bypasses type.__setattr__, so the attribute + // cache has to be invalidated by hand or lookups may return the old value. + PyType_Modified(type); + + Py_RETURN_NONE; +} + static MagicMethodLookupResult get_magic_method_slot(PyTypeObject *type, char const *name_str) { std::string name{name_str}; @@ -416,7 +571,7 @@ static MagicMethodLookupResult get_magic_method_slot(PyTypeObject *type, char co return MagicMethodLookupResult::error(NOT_IMPLEMENTED); if (type->tp_as_number != NULL) return &(type->tp_as_number->nb_inplace_add); - return &type->tp_as_sequence->sq_inplace_repeat; + return &type->tp_as_sequence->sq_inplace_concat; } SLOT(__isub__, nb_inplace_subtract) if (name == "__imul__") @@ -530,16 +685,37 @@ static MagicMethodLookupResult get_magic_method_implementation(char const *name_ return MagicMethodLookupResult::error(INVALID_MAGIC_METHOD); } -static PyObject *enable_magic_method(PyObject *self, PyObject *args) +// The sequence slots that take a Py_ssize_t have a different signature than the +// matching number/mapping slots, so they need implementations of their own. +static void *get_sequence_slot_implementation(PyTypeObject *type, void **slot) { - PyTypeObject *type; - PyObject *method; - if (!PyArg_ParseTuple(args, "OO", &type, &method)) + PySequenceMethods *sequence = type->tp_as_sequence; + if (sequence == NULL) { return NULL; } - char const *method_name = PyUnicode_AsUTF8(method); + if (slot == reinterpret_cast(&sequence->sq_item)) + { + return reinterpret_cast(impl_sq_item); + } + if (slot == reinterpret_cast(&sequence->sq_ass_item)) + { + return reinterpret_cast(impl_sq_ass_item); + } + if (slot == reinterpret_cast(&sequence->sq_repeat)) + { + return reinterpret_cast(impl_sq_repeat); + } + if (slot == reinterpret_cast(&sequence->sq_inplace_repeat)) + { + return reinterpret_cast(impl_sq_inplace_repeat); + } + return NULL; +} + +static PyObject *enable_magic_method_locked(PyTypeObject *type, char const *method_name) +{ auto slot_result = get_magic_method_slot(type, method_name); if (slot_result.is_error()) { @@ -560,37 +736,65 @@ static PyObject *enable_magic_method(PyObject *self, PyObject *args) void **slot = (void **)slot_result.result; - auto impl_result = get_magic_method_implementation(method_name); + if (slot == NULL) + { + PyErr_SetString(PyExc_RuntimeError, "slot was null"); + return NULL; + } + + void *impl = get_sequence_slot_implementation(type, slot); - if (impl_result.is_error()) + if (impl == NULL) { - if (impl_result.error_code == NOT_IMPLEMENTED) - { - PyErr_SetString(PyExc_NotImplementedError, (std::string{"[ExType] Method '"} + method_name + "' is not implemented").c_str()); - return NULL; - } - else + auto impl_result = get_magic_method_implementation(method_name); + + if (impl_result.is_error()) { - Py_RETURN_NONE; + if (impl_result.error_code == NOT_IMPLEMENTED) + { + PyErr_SetString(PyExc_NotImplementedError, (std::string{"[ExType] Method '"} + method_name + "' is not implemented").c_str()); + return NULL; + } + else + { + Py_RETURN_NONE; + } } + + impl = impl_result.result; } - void *impl = impl_result.result; + STORE_SLOT(slot, impl); + PyType_Modified(type); - if (slot == NULL) + Py_RETURN_NONE; +} + +static PyObject *enable_magic_method(PyObject *self, PyObject *args) +{ + PyTypeObject *type; + PyObject *method; + if (!PyArg_ParseTuple(args, "O!U", &PyType_Type, &type, &method)) + { + return NULL; + } + + char const *method_name = PyUnicode_AsUTF8(method); + if (method_name == NULL) { - PyErr_SetString(PyExc_RuntimeError, "slot was null"); - PyErr_SetString(PyExc_RuntimeError, "slot was null"); return NULL; } - *slot = impl; + TYPE_MUTATION_LOCK(); + PyObject *result = enable_magic_method_locked(type, method_name); + TYPE_MUTATION_UNLOCK(); - Py_RETURN_NONE; + return result; } static PyMethodDef ExTypeMethods[] = { {"get_type_dict", get_type_dict, METH_VARARGS, "get the given type's tp_dict slot"}, + {"set_type_attribute", set_type_attribute, METH_VARARGS, "set an attribute directly in the given type's dict"}, {"implement_protocol_on_type", implement_protocol_on_type, METH_VARARGS, "implement protocols by the given flag"}, {"enable_magic_method", enable_magic_method, METH_VARARGS, "enable magic method on"}, {NULL, NULL, 0, NULL} /* Sentinel */ @@ -606,5 +810,16 @@ static PyModuleDef ExTypeModule{ PyMODINIT_FUNC PyInit_core(void) { - return PyModule_Create(&ExTypeModule); + PyObject *module = PyModule_Create(&ExTypeModule); + +#ifdef Py_GIL_DISABLED + // Tell the interpreter not to re-enable the GIL when this module is imported + if (module != NULL && PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED) < 0) + { + Py_DECREF(module); + return NULL; + } +#endif + + return module; } diff --git a/tests/test_free_threading.py b/tests/test_free_threading.py new file mode 100644 index 0000000..4e85b37 --- /dev/null +++ b/tests/test_free_threading.py @@ -0,0 +1,88 @@ +import sys +import sysconfig +import threading + +import pytest + +from extype.extension_utils import extend_type_with, extension + + +free_threaded = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + +THREAD_COUNT = 8 + + +def run_in_threads(*targets): + """Runs all targets at the same time and re-raises the first exception any of them raised.""" + barrier = threading.Barrier(len(targets)) + errors = [] + + def run(target): + barrier.wait() + try: + target() + except BaseException as e: + errors.append(e) + + threads = [threading.Thread(target=run, args=(target,)) for target in targets] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + if errors: + raise errors[0] + + +@pytest.mark.skipif(not free_threaded, reason="requires a free-threaded build") +def test_import_keeps_gil_disabled(): + import extype.core # noqa: F401 + + assert not sys._is_gil_enabled() + + +def test_extending_while_other_threads_use_the_type(): + class Target: + pass + + class Extension: + @extension + def __add__(self, other): + return 42 + + @extension + def value(self): + return 42 + + extend_type_with(Target, Extension) + + def writer(): + for _ in range(200): + extend_type_with(Target, Extension) + + def reader(): + target = Target() + for _ in range(2000): + assert target + 1 == 42 + assert target.value() == 42 + + run_in_threads(*[writer if i % 2 else reader for i in range(THREAD_COUNT)]) + + +def test_concurrent_extensions_are_all_applied(): + class Target: + pass + + def make_extension(i): + def method(self): + return i + + return type(f"Extension{i}", (), {f"method_{i}": extension(method)}) + + extensions = [make_extension(i) for i in range(THREAD_COUNT)] + + run_in_threads(*[lambda e=e: extend_type_with(Target, e) for e in extensions]) + + target = Target() + for i in range(THREAD_COUNT): + assert getattr(target, f"method_{i}")() == i diff --git a/tests/test_slot_implementations.py b/tests/test_slot_implementations.py new file mode 100644 index 0000000..07d5bb6 --- /dev/null +++ b/tests/test_slot_implementations.py @@ -0,0 +1,186 @@ +import sys +import types + +from extype.core import implement_protocol_on_type +from extype.extension_utils import extend_type_with, extension, Protocol + + +# Each test extends a builtin type that the rest of the test suite (and pytest) doesn't use, +# since extensions on builtin types can't be undone. + + +def test_mapping_slots_pass_the_key(): + implement_protocol_on_type(types.SimpleNamespace, Protocol.Mapping) + + class MappingExtension: + @extension + def __getitem__(self, key): + return getattr(self, key) + + @extension + def __setitem__(self, key, value): + setattr(self, key, value) + + @extension + def __delitem__(self, key): + delattr(self, key) + + extend_type_with(types.SimpleNamespace, MappingExtension) + + ns = types.SimpleNamespace() + ns["a"] = 1 + assert ns.a == 1 + assert ns["a"] == 1 + del ns["a"] + assert not hasattr(ns, "a") + + +def make_cell(value): + # types.CellType only exists since Python 3.8 + return (lambda: value).__closure__[0] + + +def test_sequence_slots_with_index_arguments(): + cell_type = type(make_cell(None)) + implement_protocol_on_type(cell_type, Protocol.Sequence) + + class SequenceExtension: + @extension + def __len__(self): + return len(self.cell_contents) + + @extension + def __getitem__(self, index): + return self.cell_contents[index] + + @extension + def __setitem__(self, index, value): + self.cell_contents[index] = value + + @extension + def __delitem__(self, index): + del self.cell_contents[index] + + @extension + def __contains__(self, value): + # Any truthy value counts, like in Python + return "yes" if value in self.cell_contents else "" + + @extension + def __mul__(self, count): + return self.cell_contents * count + + @extension + def __imul__(self, count): + self.cell_contents *= count + return self + + @extension + def __iadd__(self, other): + self.cell_contents += other + return self + + extend_type_with(cell_type, SequenceExtension) + + cell = make_cell([1, 2, 3]) + assert len(cell) == 3 + assert cell[1] == 2 + cell[1] = 20 + assert cell.cell_contents == [1, 20, 3] + del cell[1] + assert cell.cell_contents == [1, 3] + assert 3 in cell + assert 2 not in cell + assert cell * 2 == [1, 3, 1, 3] + + original = cell + cell *= 2 + assert cell is original + assert cell.cell_contents == [1, 3, 1, 3] + cell += [5] + assert cell is original + assert cell.cell_contents == [1, 3, 1, 3, 5] + + +def test_index_slot_is_unary(): + ellipsis_type = type(...) + implement_protocol_on_type(ellipsis_type, Protocol.Number) + + class IndexExtension: + @extension + def __index__(self): + return 2 + + extend_type_with(ellipsis_type, IndexExtension) + + index = ... + assert [10, 20, 30][index] == 30 + + +def test_hash_of_minus_one_becomes_minus_two(): + class Target: + pass + + class HashExtension: + @extension + def __hash__(self): + return -1 + + extend_type_with(Target, HashExtension) + + assert hash(Target()) == -2 + + +def test_reflected_operator_after_not_implemented(): + class Left: + pass + + class Right: + def __radd__(self, other): + return "radd" + + class AddExtension: + @extension + def __add__(self, other): + return NotImplemented + + extend_type_with(Left, AddExtension) + + assert Left() + Right() == "radd" + + +def test_slots_do_not_leak_references(): + class Target: + pass + + class ReprExtension: + @extension + def __repr__(self): + return "target" + + extend_type_with(Target, ReprExtension) + + target = Target() + before = sys.getrefcount(target) + for _ in range(1000): + repr(target) + assert sys.getrefcount(target) == before + + +def test_extending_invalidates_attribute_cache(): + class Target: + def value(self): + return 1 + + target = Target() + for _ in range(100): + assert target.value() == 1 + + class ValueExtension: + @extension + def value(self): + return 2 + + extend_type_with(Target, ValueExtension) + + assert target.value() == 2