From f54c5fc6c0394ea322d82aed3e338544251ec7f9 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sat, 1 Mar 2025 09:43:58 -0500 Subject: [PATCH 01/32] slow arraymodule --- Lib/test/test_array.py | 265 +++++++++++++++++ Modules/arraymodule.c | 520 +++++++++++++++++++++++---------- Modules/clinic/arraymodule.c.h | 156 +++++++++- 3 files changed, 782 insertions(+), 159 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 58ea89c4fac833..a7d2105b938598 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -3,16 +3,20 @@ """ import collections.abc +import io import unittest from test import support from test.support import import_helper from test.support import os_helper +from test.support import threading_helper from test.support import _2G import weakref import pickle import operator +import random import struct import sys +import threading import warnings import array @@ -1673,5 +1677,266 @@ def test_gh_128961(self): self.assertRaises(StopIteration, next, it) +class FreeThreadingTest(unittest.TestCase): + # Test pretty much everything that can break under free-threading. + # Non-deterministic, but at least one of these things will fail if + # array module is not free-thread safe. + + @unittest.skipUnless(support.Py_GIL_DISABLED, 'this test can only possibly fail with GIL disabled') + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_free_threading(self): + def pop1(b, a): # MODIFIES! + b.wait() + try: a.pop() + except IndexError: pass + + def append1(b, a): # MODIFIES! + b.wait() + a.append(2) + + def insert1(b, a): # MODIFIES! + b.wait() + a.insert(0, 2) + + def extend(b, a): # MODIFIES! + c = array.array('i', [2]) + b.wait() + a.extend(c) + + def extend2(b, a, c): # MODIFIES! + b.wait() + a.extend(c) + + def inplace_concat(b, a): # MODIFIES! + c = array.array('i', [2]) + b.wait() + a += c + + def inplace_concat2(b, a, c): # MODIFIES! + b.wait() + a += c + + def inplace_repeat2(b, a): # MODIFIES! + b.wait() + a *= 2 + + def clear(b, a, *args): # MODIFIES! + b.wait() + a.clear() + + def clear2(b, a, c): # MODIFIES c! + b.wait() + try: c.clear() + except BufferError: pass + + def remove1(b, a): # MODIFIES! + b.wait() + try: a.remove(1) + except ValueError: pass + + def fromunicode(b, a): # MODIFIES! + b.wait() + a.fromunicode('test') + + def frombytes(b, a): # MODIFIES! + b.wait() + a.frombytes(b'0000') + + def frombytes2(b, a, c): # MODIFIES! + b.wait() + a.frombytes(c) + + def fromlist(b, a): # MODIFIES! + n = random.randint(0, 100) + b.wait() + a.fromlist([2] * n) + + def ass_subscr2(b, a, c): # MODIFIES! + b.wait() + a[:] = c + + def ass0(b, a): # modifies inplace + b.wait() + try: a[0] = 0 + except IndexError: pass + + def byteswap(b, a): # modifies inplace + b.wait() + a.byteswap() + + def tounicode(b, a): + b.wait() + a.tounicode() + + def tobytes(b, a): + b.wait() + a.tobytes() + + def tolist(b, a): + b.wait() + a.tolist() + + def tofile(b, a): + f = io.BytesIO() + b.wait() + a.tofile(f) + + def reduce_ex2(b, a): + b.wait() + a.__reduce_ex__(2) + + def reduce_ex3(b, a): + b.wait() + c = a.__reduce_ex__(3) + assert not c[1] or 0xdd not in c[1][3] + + def copy(b, a): + b.wait() + c = a.__copy__() + assert not c or 0xdd not in c + + def repr1(b, a): + b.wait() + repr(a) + + def repeat2(b, a): + b.wait() + a * 2 + + def count1(b, a): + b.wait() + a.count(1) + + def index1(b, a): + b.wait() + try: a.index(1) + except ValueError: pass + + def contains1(b, a): + b.wait() + try: 1 in a + except ValueError: pass + + def subscr0(b, a): + b.wait() + try: a[0] + except IndexError: pass + + def concat(b, a): + b.wait() + a + a + + def concat2(b, a, c): + b.wait() + a + c + + def richcmplhs(b, a): + c = a[:] + b.wait() + a == c + + def richcmprhs(b, a): + c = a[:] + b.wait() + c == a + + def new(b, a): + tc = a.typecode + b.wait() + array.array(tc, a) + + def repr_(b, a): + b.wait() + repr(a) + + def irepeat(b, a): # MODIFIES! + b.wait() + a *= 2 + + def newi(b, l): + b.wait() + array.array('i', l) + + def fromlistl(b, a, l): # MODIFIES! + b.wait() + a.fromlist(l) + + def fromlistlclear(b, a, l): # MODIFIES LIST! + b.wait() + l.clear() + + def iter_next(b, a, it): # MODIFIES ITERATOR! + b.wait() + list(it) + + def iter_reduce(b, a, it): + b.wait() + c = it.__reduce__() + assert not c[1] or 0xdd not in c[1][0] + + def check(funcs, a=None, *args): + if a is None: + a = array.array('i', [1]) + + barrier = threading.Barrier(len(funcs)) + threads = [] + + for func in funcs: + thread = threading.Thread(target=func, args=(barrier, a, *args)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + check([pop1] * 10) + check([pop1] + [subscr0] * 10) + check([append1] * 10) + check([insert1] * 10) + check([pop1] + [index1] * 10) + check([pop1] + [contains1] * 10) + check([insert1] + [repeat2] * 10) + check([pop1] + [repr1] * 10) + check([inplace_repeat2] * 10) + check([byteswap] * 10) + check([insert1] + [clear] * 10) + check([pop1] + [count1] * 10) + check([remove1] * 10) + check([clear] + [copy] * 10, array.array('B', b'0' * 0x400000)) + check([pop1] + [reduce_ex2] * 10) + check([clear] + [reduce_ex3] * 10, array.array('B', b'0' * 0x400000)) + check([pop1] + [tobytes] * 10) + check([pop1] + [tolist] * 10) + check([clear, tounicode] * 10, array.array('w', 'a'*10000)) + check([clear, tofile] * 10, array.array('w', 'a'*10000)) + check([clear] + [extend] * 10) + check([clear] + [inplace_concat] * 10) + check([clear] + [concat] * 10, array.array('w', 'a'*10000)) + check([fromunicode] * 10, array.array('w', 'a')) + check([frombytes] * 10) + check([fromlist] * 10) + check([clear] + [richcmplhs] * 10, array.array('i', [1]*10000)) + check([clear] + [richcmprhs] * 10, array.array('i', [1]*10000)) + check([clear, ass0] * 10, array.array('i', [1]*10000)) # to test array_ass_item must disable Py_mp_ass_subscript + check([clear] + [new] * 10, array.array('w', 'a'*10000)) + check([clear] + [repr_] * 10, array.array('B', b'0' * 0x40000)) + check([clear] + [repr_] * 10, array.array('B', b'0' * 0x40000)) + check([clear] + [irepeat] * 10, array.array('B', b'0' * 0x40000)) + check([clear] + [iter_reduce] * 10, a := array.array('B', b'0' * 0x400), iter(a)) + + # make sure we handle non-self objects correctly + check([clear] + [newi] * 10, [2] * random.randint(0, 100)) + check([fromlistlclear] + [fromlistl] * 10, array.array('i', [1]), [2] * random.randint(0, 100)) + check([clear2] + [concat2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + check([clear2] + [inplace_concat2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + check([clear2] + [extend2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + check([clear2] + [ass_subscr2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + check([clear2] + [frombytes2] * 10, array.array('w', 'a'*10000), array.array('B', b'a'*10000)) + + # iterator stuff + check([clear] + [iter_next] * 10, a := array.array('i', [1] * 10), iter(a)) + + if __name__ == "__main__": unittest.main() diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 5b86ec98393e48..0775b26e1d68ed 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -13,6 +13,7 @@ #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_pyatomic_ft_wrappers.h" #include // offsetof() #include @@ -68,6 +69,19 @@ typedef struct { PyObject *str_iter; } array_state; +static inline Py_ssize_t Pyarrayobject_GET_SIZE(PyObject *op) { + arrayobject *ao = (arrayobject *)op; +#ifdef Py_GIL_DISABLED + return _Py_atomic_load_ssize_relaxed(&(_PyVarObject_CAST(ao)->ob_size)); +#else + return Py_SIZE(ao); +#endif +} +#define Pyarrayobject_GET_SIZE(op) Pyarrayobject_GET_SIZE(_PyObject_CAST(op)) + +/* Forward declaration. */ +static PyObject *array_array_frombytes(PyObject *self, PyObject *bytes); + static array_state * get_array_state(PyObject *module) { @@ -133,6 +147,7 @@ enum machine_format_code { static int array_resize(arrayobject *self, Py_ssize_t newsize) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); char *items; size_t _new_size; @@ -158,7 +173,7 @@ array_resize(arrayobject *self, Py_ssize_t newsize) PyMem_Free(self->ob_item); self->ob_item = NULL; Py_SET_SIZE(self, 0); - self->allocated = 0; + FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, 0); return 0; } @@ -188,7 +203,7 @@ array_resize(arrayobject *self, Py_ssize_t newsize) } self->ob_item = items; Py_SET_SIZE(self, newsize); - self->allocated = _new_size; + FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, _new_size); return 0; } @@ -672,6 +687,7 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *des static PyObject * getarrayitem(PyObject *op, Py_ssize_t i) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); #ifndef NDEBUG array_state *state = find_array_state_by_type(Py_TYPE(op)); assert(array_Check(op, state)); @@ -685,6 +701,7 @@ getarrayitem(PyObject *op, Py_ssize_t i) static int ins1(arrayobject *self, Py_ssize_t where, PyObject *v) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); char *items; Py_ssize_t n = Py_SIZE(self); if (v == NULL) { @@ -728,6 +745,11 @@ array_dealloc(PyObject *op) PyObject_GC_UnTrack(op); arrayobject *self = arrayobject_CAST(op); + if (self->ob_exports > 0) { + PyErr_SetString(PyExc_SystemError, + "deallocated array object has exported buffers"); + PyErr_Print(); + } if (self->weakreflist != NULL) { PyObject_ClearWeakRefs(op); } @@ -739,8 +761,10 @@ array_dealloc(PyObject *op) } static PyObject * -array_richcompare(PyObject *v, PyObject *w, int op) +array_richcompare_lock_held(PyObject *v, PyObject *w, int op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(v); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(w); array_state *state = find_array_state_by_type(Py_TYPE(v)); arrayobject *va, *wa; PyObject *vi = NULL; @@ -848,15 +872,27 @@ array_richcompare(PyObject *v, PyObject *w, int op) return res; } +static PyObject * +array_richcompare(PyObject *v, PyObject *w, int op) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION2(v, w); + ret = array_richcompare_lock_held(v, w, op); + Py_END_CRITICAL_SECTION2(); + return ret; +} + static Py_ssize_t array_length(PyObject *op) { - return Py_SIZE(op); + arrayobject *self = arrayobject_CAST(op); + return Pyarrayobject_GET_SIZE(self); } static PyObject * -array_item(PyObject *op, Py_ssize_t i) +array_item_lock_held(PyObject *op, Py_ssize_t i) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); if (i < 0 || i >= Py_SIZE(op)) { PyErr_SetString(PyExc_IndexError, "array index out of range"); return NULL; @@ -864,9 +900,20 @@ array_item(PyObject *op, Py_ssize_t i) return getarrayitem(op, i); } +static PyObject * +array_item(PyObject *op, Py_ssize_t i) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_item_lock_held(op, i); + Py_END_CRITICAL_SECTION(); + return ret; +} + static PyObject * array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(a); array_state *state = find_array_state_by_type(Py_TYPE(a)); arrayobject *np; @@ -891,6 +938,7 @@ array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) } /*[clinic input] +@critical_section array.array.clear Remove all items from the array. @@ -898,7 +946,7 @@ Remove all items from the array. static PyObject * array_array_clear_impl(arrayobject *self) -/*[clinic end generated code: output=5efe0417062210a9 input=5dffa30e94e717a4]*/ +/*[clinic end generated code: output=5efe0417062210a9 input=1c9dfcc80f5b6731]*/ { if (array_resize(self, 0) == -1) { return NULL; @@ -907,6 +955,7 @@ array_array_clear_impl(arrayobject *self) } /*[clinic input] +@critical_section array.array.__copy__ Return a copy of the array. @@ -914,12 +963,13 @@ Return a copy of the array. static PyObject * array_array___copy___impl(arrayobject *self) -/*[clinic end generated code: output=dec7c3f925d9619e input=ad1ee5b086965f09]*/ +/*[clinic end generated code: output=dec7c3f925d9619e input=7622f8f9489472d5]*/ { return array_slice(self, 0, Py_SIZE(self)); } /*[clinic input] +@critical_section array.array.__deepcopy__ unused: object @@ -929,15 +979,17 @@ Return a copy of the array. [clinic start generated code]*/ static PyObject * -array_array___deepcopy__(arrayobject *self, PyObject *unused) -/*[clinic end generated code: output=1ec748d8e14a9faa input=2405ecb4933748c4]*/ +array_array___deepcopy___impl(arrayobject *self, PyObject *unused) +/*[clinic end generated code: output=703b4c412feaaf31 input=1a29f718f5b8a1dc]*/ { return array_array___copy___impl(self); } static PyObject * -array_concat(PyObject *op, PyObject *bb) +array_concat_lock_held(PyObject *op, PyObject *bb) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(bb); arrayobject *a = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(a)); Py_ssize_t size; @@ -973,8 +1025,19 @@ array_concat(PyObject *op, PyObject *bb) } static PyObject * -array_repeat(PyObject *op, Py_ssize_t n) +array_concat(PyObject *op, PyObject *bb) { + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION2(op, bb); + ret = array_concat_lock_held(op, bb); + Py_END_CRITICAL_SECTION2(); + return ret; +} + +static PyObject * +array_repeat_lock_held(PyObject *op, Py_ssize_t n) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); arrayobject *a = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(a)); @@ -998,9 +1061,20 @@ array_repeat(PyObject *op, Py_ssize_t n) return (PyObject *)np; } +static PyObject * +array_repeat(PyObject *op, Py_ssize_t n) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_repeat_lock_held(op, n); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int array_del_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(a); char *item; Py_ssize_t d; /* Change in size */ if (ilow < 0) @@ -1034,8 +1108,9 @@ array_del_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) } static int -array_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) +setarrayitem(PyObject *op, Py_ssize_t i, PyObject *v) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); arrayobject *a = arrayobject_CAST(op); if (i < 0 || i >= Py_SIZE(a)) { PyErr_SetString(PyExc_IndexError, @@ -1048,18 +1123,20 @@ array_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) } static int -setarrayitem(PyObject *a, Py_ssize_t i, PyObject *v) +array_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) { -#ifndef NDEBUG - array_state *state = find_array_state_by_type(Py_TYPE(a)); - assert(array_Check(a, state)); -#endif - return array_ass_item(a, i, v); + int ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = setarrayitem(op, i, v); + Py_END_CRITICAL_SECTION(); + return ret; } static int array_iter_extend(arrayobject *self, PyObject *bb) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(bb); PyObject *it, *v; it = PyObject_GetIter(bb); @@ -1081,8 +1158,10 @@ array_iter_extend(arrayobject *self, PyObject *bb) } static int -array_do_extend(array_state *state, arrayobject *self, PyObject *bb) +array_do_extend_lock_held(array_state *state, arrayobject *self, PyObject *bb) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(bb); Py_ssize_t size, oldsize, bbsize; if (!array_Check(bb, state)) @@ -1113,6 +1192,16 @@ array_do_extend(array_state *state, arrayobject *self, PyObject *bb) #undef b } +static int +array_do_extend(array_state *state, arrayobject *self, PyObject *bb) +{ + int ret; + Py_BEGIN_CRITICAL_SECTION2(self, bb); + ret = array_do_extend_lock_held(state, self, bb); + Py_END_CRITICAL_SECTION2(); + return ret; +} + static PyObject * array_inplace_concat(PyObject *op, PyObject *bb) { @@ -1131,8 +1220,9 @@ array_inplace_concat(PyObject *op, PyObject *bb) } static PyObject * -array_inplace_repeat(PyObject *op, Py_ssize_t n) +array_inplace_repeat_lock_held(PyObject *op, Py_ssize_t n) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); arrayobject *self = arrayobject_CAST(op); const Py_ssize_t array_size = Py_SIZE(self); @@ -1155,16 +1245,19 @@ array_inplace_repeat(PyObject *op, Py_ssize_t n) return Py_NewRef(self); } - static PyObject * -ins(arrayobject *self, Py_ssize_t where, PyObject *v) +array_inplace_repeat(PyObject *op, Py_ssize_t n) { - if (ins1(self, where, v) != 0) - return NULL; - Py_RETURN_NONE; + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_inplace_repeat_lock_held(op, n); + Py_END_CRITICAL_SECTION(); + return ret; } + /*[clinic input] +@critical_section array.array.count v: object @@ -1174,8 +1267,8 @@ Return number of occurrences of v in the array. [clinic start generated code]*/ static PyObject * -array_array_count(arrayobject *self, PyObject *v) -/*[clinic end generated code: output=3dd3624bf7135a3a input=d9bce9d65e39d1f5]*/ +array_array_count_impl(arrayobject *self, PyObject *v) +/*[clinic end generated code: output=93ead26a2affb739 input=c12c0042c1d0e27e]*/ { Py_ssize_t count = 0; Py_ssize_t i; @@ -1199,6 +1292,7 @@ array_array_count(arrayobject *self, PyObject *v) /*[clinic input] +@critical_section array.array.index v: object @@ -1214,7 +1308,7 @@ Raise ValueError if the value is not present. static PyObject * array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, Py_ssize_t stop) -/*[clinic end generated code: output=c45e777880c99f52 input=089dff7baa7e5a7e]*/ +/*[clinic end generated code: output=c45e777880c99f52 input=fa32ac8ec22175d6]*/ { if (start < 0) { start += Py_SIZE(self); @@ -1247,22 +1341,34 @@ array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, } static int -array_contains(PyObject *self, PyObject *v) +array_contains_lock_held(PyObject *op, PyObject *v) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); Py_ssize_t i; int cmp; - for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(self); i++) { - PyObject *selfi = getarrayitem(self, i); - if (selfi == NULL) + for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(op); i++) { + PyObject *opi = getarrayitem(op, i); + if (opi == NULL) return -1; - cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); - Py_DECREF(selfi); + cmp = PyObject_RichCompareBool(opi, v, Py_EQ); + Py_DECREF(opi); } return cmp; } +static int +array_contains(PyObject *op, PyObject *v) +{ + int ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_contains_lock_held(op, v); + Py_END_CRITICAL_SECTION(); + return ret; +} + /*[clinic input] +@critical_section array.array.remove v: object @@ -1272,8 +1378,8 @@ Remove the first occurrence of v in the array. [clinic start generated code]*/ static PyObject * -array_array_remove(arrayobject *self, PyObject *v) -/*[clinic end generated code: output=bef06be9fdf9dceb input=0b1e5aed25590027]*/ +array_array_remove_impl(arrayobject *self, PyObject *v) +/*[clinic end generated code: output=f2a24e288ecb2a35 input=78bef3fd40e62f7a]*/ { Py_ssize_t i; @@ -1299,6 +1405,7 @@ array_array_remove(arrayobject *self, PyObject *v) } /*[clinic input] +@critical_section array.array.pop i: Py_ssize_t = -1 @@ -1311,7 +1418,7 @@ i defaults to -1. static PyObject * array_array_pop_impl(arrayobject *self, Py_ssize_t i) -/*[clinic end generated code: output=bc1f0c54fe5308e4 input=8e5feb4c1a11cd44]*/ +/*[clinic end generated code: output=bc1f0c54fe5308e4 input=c69a7f1f8c570e2f]*/ { PyObject *v; @@ -1358,6 +1465,7 @@ array_array_extend_impl(arrayobject *self, PyTypeObject *cls, PyObject *bb) } /*[clinic input] +@critical_section array.array.insert i: Py_ssize_t @@ -1369,12 +1477,15 @@ Insert a new item v into the array before position i. static PyObject * array_array_insert_impl(arrayobject *self, Py_ssize_t i, PyObject *v) -/*[clinic end generated code: output=5a3648e278348564 input=5577d1b4383e9313]*/ +/*[clinic end generated code: output=5a3648e278348564 input=3c922bbd81462978]*/ { - return ins(self, i, v); + if (ins1(self, i, v) != 0) + return NULL; + Py_RETURN_NONE; } /*[clinic input] +@critical_section array.array.buffer_info Return a tuple (address, length) giving the current memory address and the length in items of the buffer used to hold array's contents. @@ -1385,7 +1496,7 @@ the buffer length in bytes. static PyObject * array_array_buffer_info_impl(arrayobject *self) -/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=a58bae5c6e1ac6a6]*/ +/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=9d0dc1ff0e6542e8]*/ { PyObject *retval = NULL, *v; @@ -1411,6 +1522,7 @@ array_array_buffer_info_impl(arrayobject *self) } /*[clinic input] +@critical_section array.array.append v: object @@ -1420,13 +1532,16 @@ Append new value v to the end of the array. [clinic start generated code]*/ static PyObject * -array_array_append(arrayobject *self, PyObject *v) -/*[clinic end generated code: output=745a0669bf8db0e2 input=0b98d9d78e78f0fa]*/ +array_array_append_impl(arrayobject *self, PyObject *v) +/*[clinic end generated code: output=2f1e8cbad70c2a8b input=9cdd897c66a40c3f]*/ { - return ins(self, Py_SIZE(self), v); + if (ins1(self, Py_SIZE(self), v) != 0) + return NULL; + Py_RETURN_NONE; } /*[clinic input] +@critical_section array.array.byteswap Byteswap all items of the array. @@ -1437,7 +1552,7 @@ raised. static PyObject * array_array_byteswap_impl(arrayobject *self) -/*[clinic end generated code: output=5f8236cbdf0d90b5 input=6a85591b950a0186]*/ +/*[clinic end generated code: output=5f8236cbdf0d90b5 input=e691b6eff94d8b2e]*/ { char *p; Py_ssize_t i; @@ -1487,6 +1602,7 @@ array_array_byteswap_impl(arrayobject *self) } /*[clinic input] +@critical_section array.array.reverse Reverse the order of the items in the array. @@ -1494,7 +1610,7 @@ Reverse the order of the items in the array. static PyObject * array_array_reverse_impl(arrayobject *self) -/*[clinic end generated code: output=c04868b36f6f4089 input=cd904f01b27d966a]*/ +/*[clinic end generated code: output=c04868b36f6f4089 input=e3947e98aed068ed]*/ { Py_ssize_t itemsize = self->ob_descr->itemsize; char *p, *q; @@ -1584,6 +1700,7 @@ array_array_fromfile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f, } /*[clinic input] +@critical_section array.array.tofile cls: defining_class @@ -1595,7 +1712,7 @@ Write all items (as machine values) to the file object f. static PyObject * array_array_tofile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f) -/*[clinic end generated code: output=4560c628d9c18bc2 input=5a24da7a7b407b52]*/ +/*[clinic end generated code: output=4560c628d9c18bc2 input=a26bc66df57864dd]*/ { Py_ssize_t nbytes = Py_SIZE(self) * self->ob_descr->itemsize; /* Write 64K blocks at a time */ @@ -1628,11 +1745,12 @@ array_array_tofile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f) Py_DECREF(res); /* drop write result */ } - done: +done: Py_RETURN_NONE; } /*[clinic input] +@critical_section self list array.array.fromlist list: object @@ -1642,8 +1760,8 @@ Append items to array from list. [clinic start generated code]*/ static PyObject * -array_array_fromlist(arrayobject *self, PyObject *list) -/*[clinic end generated code: output=26411c2d228a3e3f input=be2605a96c49680f]*/ +array_array_fromlist_impl(arrayobject *self, PyObject *list) +/*[clinic end generated code: output=6c23733a68dd68df input=c7c056aaf85d997a]*/ { Py_ssize_t n; @@ -1676,6 +1794,7 @@ array_array_fromlist(arrayobject *self, PyObject *list) } /*[clinic input] +@critical_section array.array.tolist Convert array to an ordinary list with the same items. @@ -1683,7 +1802,7 @@ Convert array to an ordinary list with the same items. static PyObject * array_array_tolist_impl(arrayobject *self) -/*[clinic end generated code: output=00b60cc9eab8ef89 input=a8d7784a94f86b53]*/ +/*[clinic end generated code: output=00b60cc9eab8ef89 input=4543fdbac475c52c]*/ { PyObject *list = PyList_New(Py_SIZE(self)); Py_ssize_t i; @@ -1703,19 +1822,29 @@ array_array_tolist_impl(arrayobject *self) return NULL; } + +/*[clinic input] +@critical_section +array.array.frombytes + + buffer: Py_buffer + / + +Appends items from the string, interpreting it as an array of machine values, as if it had been read from a file using the fromfile() method. +[clinic start generated code]*/ + static PyObject * -frombytes(arrayobject *self, Py_buffer *buffer) +array_array_frombytes_impl(arrayobject *self, Py_buffer *buffer) +/*[clinic end generated code: output=d9842c8f7510a516 input=2245f9ea58579960]*/ { int itemsize = self->ob_descr->itemsize; Py_ssize_t n; if (buffer->itemsize != 1) { - PyBuffer_Release(buffer); PyErr_SetString(PyExc_TypeError, "a bytes-like object is required"); return NULL; } n = buffer->len; if (n % itemsize != 0) { - PyBuffer_Release(buffer); PyErr_SetString(PyExc_ValueError, "bytes length not a multiple of item size"); return NULL; @@ -1725,37 +1854,19 @@ frombytes(arrayobject *self, Py_buffer *buffer) Py_ssize_t old_size = Py_SIZE(self); if ((n > PY_SSIZE_T_MAX - old_size) || ((old_size + n) > PY_SSIZE_T_MAX / itemsize)) { - PyBuffer_Release(buffer); return PyErr_NoMemory(); } if (array_resize(self, old_size + n) == -1) { - PyBuffer_Release(buffer); return NULL; } memcpy(self->ob_item + old_size * itemsize, buffer->buf, n * itemsize); } - PyBuffer_Release(buffer); Py_RETURN_NONE; } /*[clinic input] -array.array.frombytes - - buffer: Py_buffer - / - -Appends items from the string, interpreting it as an array of machine values, as if it had been read from a file using the fromfile() method. -[clinic start generated code]*/ - -static PyObject * -array_array_frombytes_impl(arrayobject *self, Py_buffer *buffer) -/*[clinic end generated code: output=d9842c8f7510a516 input=378db226dfac949e]*/ -{ - return frombytes(self, buffer); -} - -/*[clinic input] +@critical_section array.array.tobytes Convert the array to an array of machine values and return the bytes representation. @@ -1763,7 +1874,7 @@ Convert the array to an array of machine values and return the bytes representat static PyObject * array_array_tobytes_impl(arrayobject *self) -/*[clinic end generated code: output=87318e4edcdc2bb6 input=90ee495f96de34f5]*/ +/*[clinic end generated code: output=87318e4edcdc2bb6 input=c4d44d5499d2320f]*/ { if (Py_SIZE(self) <= PY_SSIZE_T_MAX / self->ob_descr->itemsize) { return PyBytes_FromStringAndSize(self->ob_item, @@ -1774,6 +1885,7 @@ array_array_tobytes_impl(arrayobject *self) } /*[clinic input] +@critical_section array.array.fromunicode ustr: unicode @@ -1788,7 +1900,7 @@ some other type. static PyObject * array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) -/*[clinic end generated code: output=24359f5e001a7f2b input=025db1fdade7a4ce]*/ +/*[clinic end generated code: output=24359f5e001a7f2b input=01e2a776cee82011]*/ { int typecode = self->ob_descr->typecode; if (typecode != 'u' && typecode != 'w') { @@ -1836,6 +1948,7 @@ array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) } /*[clinic input] +@critical_section array.array.tounicode Extends this array with data from the unicode string ustr. @@ -1847,7 +1960,7 @@ unicode string from an array of some other type. static PyObject * array_array_tounicode_impl(arrayobject *self) -/*[clinic end generated code: output=08e442378336e1ef input=127242eebe70b66d]*/ +/*[clinic end generated code: output=08e442378336e1ef input=6c69dfe81a279b91]*/ { int typecode = self->ob_descr->typecode; if (typecode != 'u' && typecode != 'w') { @@ -1876,7 +1989,8 @@ array_array___sizeof___impl(arrayobject *self) /*[clinic end generated code: output=d8e1c61ebbe3eaed input=805586565bf2b3c6]*/ { size_t res = _PyObject_SIZE(Py_TYPE(self)); - res += (size_t)self->allocated * (size_t)self->ob_descr->itemsize; + res += (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(self->allocated) + * (size_t)self->ob_descr->itemsize; return PyLong_FromSize_t(res); } @@ -2271,6 +2385,7 @@ array__array_reconstructor_impl(PyObject *module, PyTypeObject *arraytype, } /*[clinic input] +@critical_section array.array.__reduce_ex__ cls: defining_class @@ -2283,7 +2398,7 @@ Return state information for pickling. static PyObject * array_array___reduce_ex___impl(arrayobject *self, PyTypeObject *cls, PyObject *value) -/*[clinic end generated code: output=4958ee5d79452ad5 input=19968cf0f91d3eea]*/ +/*[clinic end generated code: output=4958ee5d79452ad5 input=18c90a4cad7ac527]*/ { PyObject *dict; PyObject *result; @@ -2411,8 +2526,9 @@ static PyMethodDef array_methods[] = { }; static PyObject * -array_repr(PyObject *op) +array_repr_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); char typecode; PyObject *s, *v = NULL; Py_ssize_t len; @@ -2438,9 +2554,20 @@ array_repr(PyObject *op) return s; } +static PyObject * +array_repr(PyObject *op) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_repr_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; +} + static PyObject* -array_subscr(PyObject *op, PyObject *item) +array_subscr_lock_held(PyObject *op, PyObject *item) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); arrayobject *self = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(self)); @@ -2502,12 +2629,28 @@ array_subscr(PyObject *op, PyObject *item) } } +static PyObject * +array_subscr(PyObject *op, PyObject *item) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_subscr_lock_held(op, item); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int -array_ass_subscr(PyObject *op, PyObject *item, PyObject *value) +array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) { - Py_ssize_t start, stop, step, slicelength, needed; + array_state* state = find_array_state_by_type(Py_TYPE(op)); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); +#ifdef Py_DEBUG + if (value != NULL && array_Check(value, state)) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(value); + } +#endif arrayobject *self = arrayobject_CAST(op); - array_state* state = find_array_state_by_type(Py_TYPE(self)); + Py_ssize_t start, stop, step, slicelength, needed; arrayobject* other; int itemsize; @@ -2558,7 +2701,7 @@ array_ass_subscr(PyObject *op, PyObject *item, PyObject *value) value = array_slice(other, 0, needed); if (value == NULL) return -1; - ret = array_ass_subscr(op, item, value); + ret = array_ass_subscr_lock_held(op, item, value); Py_DECREF(value); return ret; } @@ -2661,19 +2804,38 @@ array_ass_subscr(PyObject *op, PyObject *item, PyObject *value) } } +static int +array_ass_subscr(PyObject *op, PyObject* item, PyObject* value) +{ + int ret; + array_state* state = find_array_state_by_type(Py_TYPE(op)); + if (value != NULL && array_Check(value, state)) { + Py_BEGIN_CRITICAL_SECTION2(op, value); + ret = array_ass_subscr_lock_held(op, item, value); + Py_END_CRITICAL_SECTION2(); + } + else { + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_ass_subscr_lock_held(op, item, value); + Py_END_CRITICAL_SECTION(); + } + return ret; +} + static const void *emptybuf = ""; static int -array_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) +array_buffer_getbuf_lock_held(PyObject *op, Py_buffer *view, int flags) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + arrayobject *self = arrayobject_CAST(op); if (view == NULL) { PyErr_SetString(PyExc_BufferError, "array_buffer_getbuf: view==NULL argument is obsolete"); return -1; } - arrayobject *self = arrayobject_CAST(op); view->buf = (void *)self->ob_item; view->obj = Py_NewRef(self); if (view->buf == NULL) @@ -2705,61 +2867,39 @@ array_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) return 0; } +static int +array_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) +{ + int ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_buffer_getbuf_lock_held(op, view, flags); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void array_buffer_relbuf(PyObject *op, Py_buffer *Py_UNUSED(view)) { + Py_BEGIN_CRITICAL_SECTION(op); arrayobject *self = arrayobject_CAST(op); self->ob_exports--; + assert(self->ob_exports >= 0); + Py_END_CRITICAL_SECTION(); } static PyObject * -array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) { +#ifdef Py_DEBUG + if (initial != NULL) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(initial); + } +#endif array_state *state = find_array_state_by_type(type); - int c; - PyObject *initial = NULL, *it = NULL; + PyObject *it = NULL; const struct arraydescr *descr; - - if ((type == state->ArrayType || - type->tp_init == state->ArrayType->tp_init) && - !_PyArg_NoKeywords("array.array", kwds)) - return NULL; - - if (!PyArg_ParseTuple(args, "C|O:array", &c, &initial)) - return NULL; - - if (PySys_Audit("array.__new__", "CO", - c, initial ? initial : Py_None) < 0) { - return NULL; - } - - if (c == 'u') { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "The 'u' type code is deprecated and " - "will be removed in Python 3.16", - 1)) { - return NULL; - } - } - bool is_unicode = c == 'u' || c == 'w'; - if (initial && !is_unicode) { - if (PyUnicode_Check(initial)) { - PyErr_Format(PyExc_TypeError, "cannot use a str to initialize " - "an array with typecode '%c'", c); - return NULL; - } - else if (array_Check(initial, state)) { - int ic = ((arrayobject*)initial)->ob_descr->typecode; - if (ic == 'u' || ic == 'w') { - PyErr_Format(PyExc_TypeError, "cannot use a unicode array to " - "initialize an array with typecode '%c'", c); - return NULL; - } - } - } - if (!(initial == NULL || PyList_Check(initial) || PyByteArray_Check(initial) || PyBytes_Check(initial) @@ -2877,6 +3017,69 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } +static PyObject * +array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + array_state *state = find_array_state_by_type(type); + int c; + PyObject *initial = NULL; + + if ((type == state->ArrayType || + type->tp_init == state->ArrayType->tp_init) && + !_PyArg_NoKeywords("array.array", kwds)) { + return NULL; + } + + if (!PyArg_ParseTuple(args, "C|O:array", &c, &initial)) { + return NULL; + } + + if (PySys_Audit("array.__new__", "CO", + c, initial ? initial : Py_None) < 0) { + return NULL; + } + + if (c == 'u') { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "The 'u' type code is deprecated and " + "will be removed in Python 3.16", + 1)) { + return NULL; + } + } + + bool is_unicode = c == 'u' || c == 'w'; + + if (initial && !is_unicode) { + if (PyUnicode_Check(initial)) { + PyErr_Format(PyExc_TypeError, "cannot use a str to initialize " + "an array with typecode '%c'", c); + return NULL; + } + else if (array_Check(initial, state)) { + int ic = ((arrayobject*)initial)->ob_descr->typecode; + if (ic == 'u' || ic == 'w') { + PyErr_Format(PyExc_TypeError, "cannot use a unicode array to " + "initialize an array with typecode '%c'", c); + return NULL; + } + } + } + + PyObject *ret; + + if (initial == NULL) { + ret = array_new_internal_lock_held(type, initial, c); + } + else { + Py_BEGIN_CRITICAL_SECTION(initial); + ret = array_new_internal_lock_held(type, initial, c); + Py_END_CRITICAL_SECTION(); + } + + return ret; +} + PyDoc_STRVAR(module_doc, "This module defines an object type which can efficiently represent\n\ @@ -3019,7 +3222,7 @@ array_iter(PyObject *op) return NULL; it->ao = (arrayobject*)Py_NewRef(ao); - it->index = 0; + it->index = 0; // -1 indicates exhausted it->getitem = ao->ob_descr->getitem; PyObject_GC_Track(it); return (PyObject *)it; @@ -3030,23 +3233,37 @@ arrayiter_next(PyObject *op) { arrayiterobject *it = arrayiterobject_CAST(op); assert(it != NULL); + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->index); + if (index < 0) { + return NULL; + } + PyObject *ret; + arrayobject *ao = it->ao; #ifndef NDEBUG array_state *state = find_array_state_by_type(Py_TYPE(it)); assert(PyObject_TypeCheck(it, state->ArrayIterType)); + assert(array_Check(ao, state)); #endif - arrayobject *ao = it->ao; - if (ao == NULL) { - return NULL; + + Py_BEGIN_CRITICAL_SECTION(ao); + if (index < Py_SIZE(ao)) { + ret = (*it->getitem)(ao, index); } -#ifndef NDEBUG - assert(array_Check(ao, state)); + else { + ret = NULL; + } + Py_END_CRITICAL_SECTION(); + + if (ret != NULL) { + FT_ATOMIC_STORE_SSIZE_RELAXED(it->index, index + 1); + } + else { + FT_ATOMIC_STORE_SSIZE_RELAXED(it->index, -1); +#ifndef Py_GIL_DISABLED + Py_CLEAR(it->ao); #endif - if (it->index < Py_SIZE(ao)) { - return (*it->getitem)(ao, it->index++); } - it->ao = NULL; - Py_DECREF(ao); - return NULL; + return ret; } static void @@ -3082,14 +3299,14 @@ static PyObject * array_arrayiterator___reduce___impl(arrayiterobject *self, PyTypeObject *cls) /*[clinic end generated code: output=4b032417a2c8f5e6 input=ac64e65a87ad452e]*/ { - array_state *state = get_array_state_by_class(cls); assert(state != NULL); PyObject *func = _PyEval_GetBuiltin(state->str_iter); - if (self->ao == NULL) { - return Py_BuildValue("N(())", func); + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->index); + if (index >= 0) { + return Py_BuildValue("N(O)n", func, self->ao, index); } - return Py_BuildValue("N(O)n", func, self->ao, self->index); + return Py_BuildValue("N(())", func); } /*[clinic input] @@ -3106,17 +3323,20 @@ array_arrayiterator___setstate__(arrayiterobject *self, PyObject *state) /*[clinic end generated code: output=397da9904e443cbe input=f47d5ceda19e787b]*/ { Py_ssize_t index = PyLong_AsSsize_t(state); - if (index == -1 && PyErr_Occurred()) + if (index == -1 && PyErr_Occurred()) { return NULL; - arrayobject *ao = self->ao; - if (ao != NULL) { - if (index < 0) { - index = 0; + } + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->index) >= 0) { + if (index < -1) { + index = -1; } - else if (index > Py_SIZE(ao)) { - index = Py_SIZE(ao); /* iterator exhausted */ + else { + Py_ssize_t size = Pyarrayobject_GET_SIZE(self->ao); + if (index > size) { + index = size; /* iterator at end */ + } } - self->index = index; + FT_ATOMIC_STORE_SSIZE_RELAXED(self->index, index); } Py_RETURN_NONE; } diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index c5b62b16699d06..3816bb7709658e 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_runtime.h" // _Py_SINGLETON() #endif #include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(array_array_clear__doc__, @@ -23,7 +24,13 @@ array_array_clear_impl(arrayobject *self); static PyObject * array_array_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_clear_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_clear_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array___copy____doc__, @@ -41,7 +48,13 @@ array_array___copy___impl(arrayobject *self); static PyObject * array_array___copy__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array___copy___impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array___copy___impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array___deepcopy____doc__, @@ -53,6 +66,21 @@ PyDoc_STRVAR(array_array___deepcopy____doc__, #define ARRAY_ARRAY___DEEPCOPY___METHODDEF \ {"__deepcopy__", (PyCFunction)array_array___deepcopy__, METH_O, array_array___deepcopy____doc__}, +static PyObject * +array_array___deepcopy___impl(arrayobject *self, PyObject *unused); + +static PyObject * +array_array___deepcopy__(arrayobject *self, PyObject *unused) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array___deepcopy___impl((arrayobject *)self, unused); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(array_array_count__doc__, "count($self, v, /)\n" "--\n" @@ -62,6 +90,21 @@ PyDoc_STRVAR(array_array_count__doc__, #define ARRAY_ARRAY_COUNT_METHODDEF \ {"count", (PyCFunction)array_array_count, METH_O, array_array_count__doc__}, +static PyObject * +array_array_count_impl(arrayobject *self, PyObject *v); + +static PyObject * +array_array_count(arrayobject *self, PyObject *v) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_count_impl((arrayobject *)self, v); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(array_array_index__doc__, "index($self, v, start=0, stop=sys.maxsize, /)\n" "--\n" @@ -102,7 +145,9 @@ array_array_index(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_index_impl((arrayobject *)self, v, start, stop); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -117,6 +162,21 @@ PyDoc_STRVAR(array_array_remove__doc__, #define ARRAY_ARRAY_REMOVE_METHODDEF \ {"remove", (PyCFunction)array_array_remove, METH_O, array_array_remove__doc__}, +static PyObject * +array_array_remove_impl(arrayobject *self, PyObject *v); + +static PyObject * +array_array_remove(arrayobject *self, PyObject *v) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_remove_impl((arrayobject *)self, v); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(array_array_pop__doc__, "pop($self, i=-1, /)\n" "--\n" @@ -156,7 +216,9 @@ array_array_pop(PyObject *self, PyObject *const *args, Py_ssize_t nargs) i = ival; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_pop_impl((arrayobject *)self, i); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -241,7 +303,9 @@ array_array_insert(PyObject *self, PyObject *const *args, Py_ssize_t nargs) i = ival; } v = args[1]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_insert_impl((arrayobject *)self, i, v); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -265,7 +329,13 @@ array_array_buffer_info_impl(arrayobject *self); static PyObject * array_array_buffer_info(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_buffer_info_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_buffer_info_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array_append__doc__, @@ -277,6 +347,21 @@ PyDoc_STRVAR(array_array_append__doc__, #define ARRAY_ARRAY_APPEND_METHODDEF \ {"append", (PyCFunction)array_array_append, METH_O, array_array_append__doc__}, +static PyObject * +array_array_append_impl(arrayobject *self, PyObject *v); + +static PyObject * +array_array_append(arrayobject *self, PyObject *v) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_append_impl((arrayobject *)self, v); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(array_array_byteswap__doc__, "byteswap($self, /)\n" "--\n" @@ -295,7 +380,13 @@ array_array_byteswap_impl(arrayobject *self); static PyObject * array_array_byteswap(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_byteswap_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_byteswap_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array_reverse__doc__, @@ -313,7 +404,13 @@ array_array_reverse_impl(arrayobject *self); static PyObject * array_array_reverse(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_reverse_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_reverse_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array_fromfile__doc__, @@ -412,7 +509,9 @@ array_array_tofile(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ goto exit; } f = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_tofile_impl((arrayobject *)self, cls, f); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -427,6 +526,21 @@ PyDoc_STRVAR(array_array_fromlist__doc__, #define ARRAY_ARRAY_FROMLIST_METHODDEF \ {"fromlist", (PyCFunction)array_array_fromlist, METH_O, array_array_fromlist__doc__}, +static PyObject * +array_array_fromlist_impl(arrayobject *self, PyObject *list); + +static PyObject * +array_array_fromlist(arrayobject *self, PyObject *list) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION2(self, list); + return_value = array_array_fromlist_impl((arrayobject *)self, list); + Py_END_CRITICAL_SECTION2(); + + return return_value; +} + PyDoc_STRVAR(array_array_tolist__doc__, "tolist($self, /)\n" "--\n" @@ -442,7 +556,13 @@ array_array_tolist_impl(arrayobject *self); static PyObject * array_array_tolist(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_tolist_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_tolist_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array_frombytes__doc__, @@ -466,7 +586,9 @@ array_array_frombytes(PyObject *self, PyObject *arg) if (PyObject_GetBuffer(arg, &buffer, PyBUF_SIMPLE) != 0) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_frombytes_impl((arrayobject *)self, &buffer); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for buffer */ @@ -492,7 +614,13 @@ array_array_tobytes_impl(arrayobject *self); static PyObject * array_array_tobytes(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_tobytes_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_tobytes_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array_fromunicode__doc__, @@ -522,7 +650,9 @@ array_array_fromunicode(PyObject *self, PyObject *arg) goto exit; } ustr = arg; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array_fromunicode_impl((arrayobject *)self, ustr); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -547,7 +677,13 @@ array_array_tounicode_impl(arrayobject *self); static PyObject * array_array_tounicode(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return array_array_tounicode_impl((arrayobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = array_array_tounicode_impl((arrayobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(array_array___sizeof____doc__, @@ -659,7 +795,9 @@ array_array___reduce_ex__(PyObject *self, PyTypeObject *cls, PyObject *const *ar goto exit; } value = args[0]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = array_array___reduce_ex___impl((arrayobject *)self, cls, value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -695,4 +833,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=8120dc5c4fa414b9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c9219e074c62e0c8 input=a9049054013a1b77]*/ From 4c71f00624330c850d71461521f3ef9fc17eab8c Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 2 Mar 2025 15:56:54 -0500 Subject: [PATCH 02/32] lockfree read and write single element --- Include/internal/pycore_pymem.h | 2 +- Modules/arraymodule.c | 538 +++++++++++++++++++++++--------- 2 files changed, 388 insertions(+), 152 deletions(-) diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 5386d4c5f83031..bcd13d80cba975 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -117,7 +117,7 @@ extern wchar_t *_PyMem_DefaultRawWcsdup(const wchar_t *str); extern int _PyMem_DebugEnabled(void); // Enqueue a pointer to be freed possibly after some delay. -extern void _PyMem_FreeDelayed(void *ptr); +PyAPI_FUNC(void) _PyMem_FreeDelayed(void *ptr); // Enqueue an object to be freed possibly after some delay #ifdef Py_GIL_DISABLED diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 0775b26e1d68ed..41a5371acaf125 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -14,6 +14,7 @@ #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyatomic_ft_wrappers.h" +#include "pycore_pymem.h" // _PyMem_FreeDelayed #include // offsetof() #include @@ -33,8 +34,8 @@ static struct PyModuleDef arraymodule; struct arraydescr { char typecode; int itemsize; - PyObject * (*getitem)(struct arrayobject *, Py_ssize_t); - int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *); + PyObject * (*getitem)(char *, Py_ssize_t); + int (*setitem)(char *, Py_ssize_t, PyObject *); int (*compareitems)(const void *, const void *, Py_ssize_t); const char *formats; int is_integer_type; @@ -54,7 +55,6 @@ typedef struct { PyObject_HEAD Py_ssize_t index; arrayobject *ao; - PyObject* (*getitem)(struct arrayobject *, Py_ssize_t); } arrayiterobject; typedef struct { @@ -144,6 +144,85 @@ enum machine_format_code { #define array_Check(op, state) PyObject_TypeCheck(op, state->ArrayType) +#ifdef Py_GIL_DISABLED +typedef struct { + Py_ssize_t allocated; + char ob_item[]; +} _PyArrayArray; +#endif + +static char * +arrayarray_alloc(Py_ssize_t size, int itemsize) +{ +#ifdef Py_GIL_DISABLED + if (size > PY_SSIZE_T_MAX/itemsize - 1) { + return NULL; + } + _PyArrayArray *array = PyMem_Malloc(sizeof(_PyArrayArray) + size * itemsize); + if (array == NULL) { + return NULL; + } + array->allocated = size; + return array->ob_item; +#else + return PyMem_Malloc(size * itemsize); +#endif +} + +static void +arrayarray_free(char *items, bool use_qsbr) +{ +#ifdef Py_GIL_DISABLED + if (items == NULL) { + return; + } + _PyArrayArray *array = _Py_CONTAINER_OF(items, _PyArrayArray, ob_item); + if (use_qsbr) { + _PyMem_FreeDelayed(array); + } + else { + PyMem_Free(array); + } +#else + PyMem_Free(items); +#endif +} + +#ifdef Py_GIL_DISABLED + +static Py_ssize_t +arrayarray_capacity(char *items) +{ + _PyArrayArray *array = _Py_CONTAINER_OF(items, _PyArrayArray, ob_item); + return array->allocated; +} + +static char * +arrayarray_set(char *items, char *newitems, Py_ssize_t newsize, int itemsize, bool use_qsbr) +{ + arrayarray_free(items, use_qsbr); + items = arrayarray_alloc(newsize, itemsize); + if (items != NULL) { + memcpy(items, newitems, newsize * itemsize); + } + return items; +} + +static void +ensure_shared_on_resize(arrayobject *self) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + // Ensure that the array is freed using QSBR if we are not the + // owning thread. + if (!_Py_IsOwnedByCurrentThread((PyObject *)self) && + !_PyObject_GC_IS_SHARED(self)) + { + _PyObject_GC_SET_SHARED(self); + } +} + +# endif // Py_GIL_DISABLED + static int array_resize(arrayobject *self, Py_ssize_t newsize) { @@ -170,7 +249,11 @@ array_resize(arrayobject *self, Py_ssize_t newsize) } if (newsize == 0) { - PyMem_Free(self->ob_item); +#ifdef Py_GIL_DISABLED + arrayarray_free(self->ob_item, _PyObject_GC_IS_SHARED(self)); +#else + arrayarray_free(self->ob_item, false); +#endif self->ob_item = NULL; Py_SET_SIZE(self, 0); FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, 0); @@ -191,17 +274,37 @@ array_resize(arrayobject *self, Py_ssize_t newsize) _new_size = (newsize >> 4) + (Py_SIZE(self) < 8 ? 3 : 7) + newsize; items = self->ob_item; + int itemsize = self->ob_descr->itemsize; + /* XXX The following multiplication and division does not optimize away like it does for lists since the size is not known at compile time */ - if (_new_size <= ((~(size_t)0) / self->ob_descr->itemsize)) - PyMem_RESIZE(items, char, (_new_size * self->ob_descr->itemsize)); - else - items = NULL; + if (_new_size > ((~(size_t)0) / itemsize)) { + PyErr_NoMemory(); + return -1; + } + +#ifdef Py_GIL_DISABLED + ensure_shared_on_resize(self); + char *newitems = arrayarray_alloc(_new_size, itemsize); + if (newitems == NULL) { + PyErr_NoMemory(); + return -1; + } + if (items != NULL) { + Py_ssize_t size = Py_SIZE(self); + memcpy(newitems, items, Py_MIN(size, newsize) * itemsize); + arrayarray_free(items, _PyObject_GC_IS_SHARED(self)); + } + _Py_atomic_store_ptr_release(&self->ob_item, newitems); +#else + PyMem_RESIZE(items, char, (_new_size * itemsize)); if (items == NULL) { PyErr_NoMemory(); return -1; } self->ob_item = items; +#endif + Py_SET_SIZE(self, newsize); FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, _new_size); return 0; @@ -220,14 +323,14 @@ in bounds; that's the responsibility of the caller. ****************************************************************************/ static PyObject * -b_getitem(arrayobject *ap, Py_ssize_t i) +b_getitem(char *ob_item, Py_ssize_t i) { - long x = ((signed char *)ap->ob_item)[i]; + long x = ((signed char *)ob_item)[i]; return PyLong_FromLong(x); } static int -b_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +b_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { short x; /* PyArg_Parse's 'b' formatter is for an unsigned char, therefore @@ -246,37 +349,37 @@ b_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((char *)ap->ob_item)[i] = (char)x; + ((char *)ob_item)[i] = (char)x; return 0; } static PyObject * -BB_getitem(arrayobject *ap, Py_ssize_t i) +BB_getitem(char *ob_item, Py_ssize_t i) { - long x = ((unsigned char *)ap->ob_item)[i]; + long x = ((unsigned char *)ob_item)[i]; return PyLong_FromLong(x); } static int -BB_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +BB_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { unsigned char x; /* 'B' == unsigned char, maps to PyArg_Parse's 'b' formatter */ if (!PyArg_Parse(v, "b;array item must be integer", &x)) return -1; if (i >= 0) - ((unsigned char *)ap->ob_item)[i] = x; + ((unsigned char *)ob_item)[i] = x; return 0; } static PyObject * -u_getitem(arrayobject *ap, Py_ssize_t i) +u_getitem(char *ob_item, Py_ssize_t i) { - return PyUnicode_FromOrdinal(((wchar_t *) ap->ob_item)[i]); + return PyUnicode_FromOrdinal(((wchar_t *) ob_item)[i]); } static int -u_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +u_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { if (!PyUnicode_Check(v)) { PyErr_Format(PyExc_TypeError, @@ -307,19 +410,19 @@ u_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) assert(len == 1); if (i >= 0) { - ((wchar_t *)ap->ob_item)[i] = w; + ((wchar_t *)ob_item)[i] = w; } return 0; } static PyObject * -w_getitem(arrayobject *ap, Py_ssize_t i) +w_getitem(char *ob_item, Py_ssize_t i) { - return PyUnicode_FromOrdinal(((Py_UCS4 *) ap->ob_item)[i]); + return PyUnicode_FromOrdinal(((Py_UCS4 *) ob_item)[i]); } static int -w_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +w_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { if (!PyUnicode_Check(v)) { PyErr_Format(PyExc_TypeError, @@ -337,38 +440,38 @@ w_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) } if (i >= 0) { - ((Py_UCS4 *)ap->ob_item)[i] = PyUnicode_READ_CHAR(v, 0); + ((Py_UCS4 *)ob_item)[i] = PyUnicode_READ_CHAR(v, 0); } return 0; } static PyObject * -h_getitem(arrayobject *ap, Py_ssize_t i) +h_getitem(char *ob_item, Py_ssize_t i) { - return PyLong_FromLong((long) ((short *)ap->ob_item)[i]); + return PyLong_FromLong((long) ((short *)ob_item)[i]); } static int -h_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +h_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { short x; /* 'h' == signed short, maps to PyArg_Parse's 'h' formatter */ if (!PyArg_Parse(v, "h;array item must be integer", &x)) return -1; if (i >= 0) - ((short *)ap->ob_item)[i] = x; + ((short *)ob_item)[i] = x; return 0; } static PyObject * -HH_getitem(arrayobject *ap, Py_ssize_t i) +HH_getitem(char *ob_item, Py_ssize_t i) { - return PyLong_FromLong((long) ((unsigned short *)ap->ob_item)[i]); + return PyLong_FromLong((long) ((unsigned short *)ob_item)[i]); } static int -HH_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +HH_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { int x; /* PyArg_Parse's 'h' formatter is for a signed short, therefore @@ -386,37 +489,37 @@ HH_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((short *)ap->ob_item)[i] = (short)x; + ((short *)ob_item)[i] = (short)x; return 0; } static PyObject * -i_getitem(arrayobject *ap, Py_ssize_t i) +i_getitem(char *ob_item, Py_ssize_t i) { - return PyLong_FromLong((long) ((int *)ap->ob_item)[i]); + return PyLong_FromLong((long) ((int *)ob_item)[i]); } static int -i_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +i_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { int x; /* 'i' == signed int, maps to PyArg_Parse's 'i' formatter */ if (!PyArg_Parse(v, "i;array item must be integer", &x)) return -1; if (i >= 0) - ((int *)ap->ob_item)[i] = x; + ((int *)ob_item)[i] = x; return 0; } static PyObject * -II_getitem(arrayobject *ap, Py_ssize_t i) +II_getitem(char *ob_item, Py_ssize_t i) { return PyLong_FromUnsignedLong( - (unsigned long) ((unsigned int *)ap->ob_item)[i]); + (unsigned long) ((unsigned int *)ob_item)[i]); } static int -II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +II_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { unsigned long x; int do_decref = 0; /* if nb_int was called */ @@ -444,7 +547,7 @@ II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((unsigned int *)ap->ob_item)[i] = (unsigned int)x; + ((unsigned int *)ob_item)[i] = (unsigned int)x; if (do_decref) { Py_DECREF(v); @@ -453,30 +556,30 @@ II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) } static PyObject * -l_getitem(arrayobject *ap, Py_ssize_t i) +l_getitem(char *ob_item, Py_ssize_t i) { - return PyLong_FromLong(((long *)ap->ob_item)[i]); + return PyLong_FromLong(((long *)ob_item)[i]); } static int -l_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +l_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { long x; if (!PyArg_Parse(v, "l;array item must be integer", &x)) return -1; if (i >= 0) - ((long *)ap->ob_item)[i] = x; + ((long *)ob_item)[i] = x; return 0; } static PyObject * -LL_getitem(arrayobject *ap, Py_ssize_t i) +LL_getitem(char *ob_item, Py_ssize_t i) { - return PyLong_FromUnsignedLong(((unsigned long *)ap->ob_item)[i]); + return PyLong_FromUnsignedLong(((unsigned long *)ob_item)[i]); } static int -LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +LL_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { unsigned long x; int do_decref = 0; /* if nb_int was called */ @@ -496,7 +599,7 @@ LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((unsigned long *)ap->ob_item)[i] = x; + ((unsigned long *)ob_item)[i] = x; if (do_decref) { Py_DECREF(v); @@ -505,31 +608,31 @@ LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) } static PyObject * -q_getitem(arrayobject *ap, Py_ssize_t i) +q_getitem(char *ob_item, Py_ssize_t i) { - return PyLong_FromLongLong(((long long *)ap->ob_item)[i]); + return PyLong_FromLongLong(((long long *)ob_item)[i]); } static int -q_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +q_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { long long x; if (!PyArg_Parse(v, "L;array item must be integer", &x)) return -1; if (i >= 0) - ((long long *)ap->ob_item)[i] = x; + ((long long *)ob_item)[i] = x; return 0; } static PyObject * -QQ_getitem(arrayobject *ap, Py_ssize_t i) +QQ_getitem(char *ob_item, Py_ssize_t i) { return PyLong_FromUnsignedLongLong( - ((unsigned long long *)ap->ob_item)[i]); + ((unsigned long long *)ob_item)[i]); } static int -QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +QQ_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { unsigned long long x; int do_decref = 0; /* if nb_int was called */ @@ -549,7 +652,7 @@ QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((unsigned long long *)ap->ob_item)[i] = x; + ((unsigned long long *)ob_item)[i] = x; if (do_decref) { Py_DECREF(v); @@ -558,36 +661,36 @@ QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) } static PyObject * -f_getitem(arrayobject *ap, Py_ssize_t i) +f_getitem(char *ob_item, Py_ssize_t i) { - return PyFloat_FromDouble((double) ((float *)ap->ob_item)[i]); + return PyFloat_FromDouble((double) ((float *)ob_item)[i]); } static int -f_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +f_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { float x; if (!PyArg_Parse(v, "f;array item must be float", &x)) return -1; if (i >= 0) - ((float *)ap->ob_item)[i] = x; + ((float *)ob_item)[i] = x; return 0; } static PyObject * -d_getitem(arrayobject *ap, Py_ssize_t i) +d_getitem(char *ob_item, Py_ssize_t i) { - return PyFloat_FromDouble(((double *)ap->ob_item)[i]); + return PyFloat_FromDouble(((double *)ob_item)[i]); } static int -d_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +d_setitem(char *ob_item, Py_ssize_t i, PyObject *v) { double x; if (!PyArg_Parse(v, "d;array item must be float", &x)) return -1; if (i >= 0) - ((double *)ap->ob_item)[i] = x; + ((double *)ob_item)[i] = x; return 0; } @@ -650,7 +753,6 @@ static PyObject * newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *descr) { arrayobject *op; - size_t nbytes; if (size < 0) { PyErr_BadInternalCall(); @@ -661,7 +763,6 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *des if (size > PY_SSIZE_T_MAX / descr->itemsize) { return PyErr_NoMemory(); } - nbytes = size * descr->itemsize; op = (arrayobject *) type->tp_alloc(type, 0); if (op == NULL) { return NULL; @@ -674,7 +775,7 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *des op->ob_item = NULL; } else { - op->ob_item = PyMem_NEW(char, nbytes); + op->ob_item = arrayarray_alloc(size, descr->itemsize); if (op->ob_item == NULL) { Py_DECREF(op); return PyErr_NoMemory(); @@ -684,20 +785,149 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *des return (PyObject *) op; } +static inline int +valid_index(Py_ssize_t i, Py_ssize_t limit) +{ + return (size_t) i < (size_t) limit; +} + static PyObject * -getarrayitem(PyObject *op, Py_ssize_t i) +getarrayitem(PyObject *op, Py_ssize_t i, char *items) +{ +#ifndef NDEBUG + array_state *state = find_array_state_by_type(Py_TYPE(op)); + assert(array_Check(op, state)); +#endif + arrayobject *ap = arrayobject_CAST(op); + assert(valid_index(i, Py_SIZE(op))); + return (*ap->ob_descr->getitem)(items, i); +} + +static PyObject * +getarrayitem_locked(PyObject *op, Py_ssize_t i) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); +#ifdef Py_GIL_DISABLED + if (!_PyObject_GC_IS_SHARED(op)) { + _PyObject_GC_SET_SHARED(op); + } +#endif + if (!valid_index(i, Py_SIZE(op))) { + return NULL; + } + arrayobject *ap = (arrayobject *)op; + ret = getarrayitem(op, i, ap->ob_item); + Py_END_CRITICAL_SECTION(); + return ret; +} + +#ifdef Py_GIL_DISABLED + +static PyObject * +getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) +{ + if (!_Py_IsOwnedByCurrentThread((PyObject *)op) && !_PyObject_GC_IS_SHARED(op)) { + return getarrayitem_locked(op, i); + } + Py_ssize_t size = Pyarrayobject_GET_SIZE(op); + if (!valid_index(i, size)) { + return NULL; + } + arrayobject *ap = (arrayobject *)op; + char *items = _Py_atomic_load_ptr(&ap->ob_item); + if (items == NULL) { + return NULL; + } + Py_ssize_t cap = arrayarray_capacity(items); + if (!valid_index(i, cap)) { + return NULL; + } + return getarrayitem(op, i, items); +} + +#else // Py_GIL_DISABLED + +static PyObject * +getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) +{ + return getarrayitem_locked(op, i); +} + +#endif // Py_GIL_DISABLED + +static int +setarrayitem(PyObject *op, Py_ssize_t i, PyObject *v, char *items) { - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); #ifndef NDEBUG array_state *state = find_array_state_by_type(Py_TYPE(op)); assert(array_Check(op, state)); #endif - arrayobject *ap; - ap = (arrayobject *)op; - assert(i>=0 && iob_descr->getitem)(ap, i); + arrayobject *ap = arrayobject_CAST(op); + assert(valid_index(i, Py_SIZE(op))); + return (*ap->ob_descr->setitem)(items, i, v); } +static int +setarrayitem_locked(PyObject *op, Py_ssize_t i, PyObject *v) +{ + int ret; + Py_BEGIN_CRITICAL_SECTION(op); +#ifdef Py_GIL_DISABLED + if (!_PyObject_GC_IS_SHARED(op)) { + _PyObject_GC_SET_SHARED(op); + } +#endif + if (!valid_index(i, Py_SIZE(op))) { + PyErr_SetString(PyExc_IndexError, "array index out of range"); + ret = -1; + } + else { + arrayobject *ap = (arrayobject *)op; + ret = setarrayitem(op, i, v, ap->ob_item); + } + Py_END_CRITICAL_SECTION(); + return ret; +} + +#ifdef Py_GIL_DISABLED + +static int +setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) +{ + if (!_Py_IsOwnedByCurrentThread((PyObject *)op) && !_PyObject_GC_IS_SHARED(op)) { + return setarrayitem_locked(op, i, v); + } + Py_ssize_t size = Pyarrayobject_GET_SIZE(op); + if (!valid_index(i, size)) { + goto error; + } + arrayobject *ap = (arrayobject *)op; + char *items = _Py_atomic_load_ptr(&ap->ob_item); + if (items == NULL) { + goto error; + } + Py_ssize_t cap = arrayarray_capacity(items); + if (!valid_index(i, cap)) { + goto error; + } + return setarrayitem(op, i, v, items); + +error: + PyErr_SetString(PyExc_IndexError, "array index out of range"); + return -1; +} + +#else // Py_GIL_DISABLED + +static int +setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) +{ + return setarrayitem_locked(op, i, v); +} + +#endif // Py_GIL_DISABLED + static int ins1(arrayobject *self, Py_ssize_t where, PyObject *v) { @@ -708,7 +938,7 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) PyErr_BadInternalCall(); return -1; } - if ((*self->ob_descr->setitem)(self, -1, v) < 0) + if ((*self->ob_descr->setitem)(self->ob_item, -1, v) < 0) return -1; if (array_resize(self, n+1) == -1) @@ -726,7 +956,7 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) memmove(items + (where+1)*self->ob_descr->itemsize, items + where*self->ob_descr->itemsize, (n-where)*self->ob_descr->itemsize); - return (*self->ob_descr->setitem)(self, where, v); + return (*self->ob_descr->setitem)(self->ob_item, where, v); } /* Methods */ @@ -754,7 +984,7 @@ array_dealloc(PyObject *op) PyObject_ClearWeakRefs(op); } if (self->ob_item != NULL) { - PyMem_Free(self->ob_item); + arrayarray_free(self->ob_item, false); } tp->tp_free(op); Py_DECREF(tp); @@ -814,11 +1044,11 @@ array_richcompare_lock_held(PyObject *v, PyObject *w, int op) /* Search for the first index where items are different */ k = 1; for (i = 0; i < Py_SIZE(va) && i < Py_SIZE(wa); i++) { - vi = getarrayitem(v, i); + vi = getarrayitem(v, i, va->ob_item); if (vi == NULL) { return NULL; } - wi = getarrayitem(w, i); + wi = getarrayitem(w, i, wa->ob_item); if (wi == NULL) { Py_DECREF(vi); return NULL; @@ -889,26 +1119,17 @@ array_length(PyObject *op) return Pyarrayobject_GET_SIZE(self); } + static PyObject * -array_item_lock_held(PyObject *op, Py_ssize_t i) +array_item(PyObject *op, Py_ssize_t i) { - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); - if (i < 0 || i >= Py_SIZE(op)) { + PyObject *item = getarrayitem_maybe_locked(op, i); + if (item == NULL) { PyErr_SetString(PyExc_IndexError, "array index out of range"); - return NULL; } - return getarrayitem(op, i); + return item; } -static PyObject * -array_item(PyObject *op, Py_ssize_t i) -{ - PyObject *ret; - Py_BEGIN_CRITICAL_SECTION(op); - ret = array_item_lock_held(op, i); - Py_END_CRITICAL_SECTION(); - return ret; -} static PyObject * array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) @@ -1107,27 +1328,15 @@ array_del_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) return 0; } -static int -setarrayitem(PyObject *op, Py_ssize_t i, PyObject *v) -{ - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); - arrayobject *a = arrayobject_CAST(op); - if (i < 0 || i >= Py_SIZE(a)) { - PyErr_SetString(PyExc_IndexError, - "array assignment index out of range"); - return -1; - } - if (v == NULL) - return array_del_slice(a, i, i+1); - return (*a->ob_descr->setitem)(a, i, v); -} - static int array_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) { + if (v != NULL) { + return setarrayitem_maybe_locked(op, i, v); + } int ret; Py_BEGIN_CRITICAL_SECTION(op); - ret = setarrayitem(op, i, v); + ret = array_del_slice(arrayobject_CAST(op), i, i+1); Py_END_CRITICAL_SECTION(); return ret; } @@ -1277,7 +1486,7 @@ array_array_count_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem((PyObject *)self, i); + selfi = getarrayitem((PyObject *)self, i, self->ob_item); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1325,7 +1534,7 @@ array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, PyObject *selfi; int cmp; - selfi = getarrayitem((PyObject *)self, i); + selfi = getarrayitem((PyObject *)self, i, self->ob_item); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1348,7 +1557,8 @@ array_contains_lock_held(PyObject *op, PyObject *v) int cmp; for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(op); i++) { - PyObject *opi = getarrayitem(op, i); + arrayobject *ap = (arrayobject *)op; + PyObject *opi = getarrayitem(op, i, ap->ob_item); if (opi == NULL) return -1; cmp = PyObject_RichCompareBool(opi, v, Py_EQ); @@ -1387,7 +1597,7 @@ array_array_remove_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem((PyObject *)self,i); + selfi = getarrayitem((PyObject *)self, i, self->ob_item); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1433,7 +1643,7 @@ array_array_pop_impl(arrayobject *self, Py_ssize_t i) PyErr_SetString(PyExc_IndexError, "pop index out of range"); return NULL; } - v = getarrayitem((PyObject *)self, i); + v = getarrayitem((PyObject *)self, i, self->ob_item); if (v == NULL) return NULL; if (array_del_slice(self, i, i+1) != 0) { @@ -1777,7 +1987,7 @@ array_array_fromlist_impl(arrayobject *self, PyObject *list) return NULL; for (i = 0; i < n; i++) { PyObject *v = PyList_GET_ITEM(list, i); - if ((*self->ob_descr->setitem)(self, + if ((*self->ob_descr->setitem)(self->ob_item, Py_SIZE(self) - n + i, v) != 0) { array_resize(self, old_size); return NULL; @@ -1810,7 +2020,7 @@ array_array_tolist_impl(arrayobject *self) if (list == NULL) return NULL; for (i = 0; i < Py_SIZE(self); i++) { - PyObject *v = getarrayitem((PyObject *)self, i); + PyObject *v = getarrayitem((PyObject *)self, i, self->ob_item); if (v == NULL) goto error; PyList_SET_ITEM(list, i, v); @@ -2565,22 +2775,13 @@ array_repr(PyObject *op) } static PyObject* -array_subscr_lock_held(PyObject *op, PyObject *item) +array_subscr_slice_lock_held(PyObject *op, PyObject *item) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); arrayobject *self = arrayobject_CAST(op); array_state *state = find_array_state_by_type(Py_TYPE(self)); - if (PyIndex_Check(item)) { - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - if (i==-1 && PyErr_Occurred()) { - return NULL; - } - if (i < 0) - i += Py_SIZE(self); - return array_item(op, i); - } - else if (PySlice_Check(item)) { + if (PySlice_Check(item)) { Py_ssize_t start, stop, step, slicelength, i; size_t cur; PyObject* result; @@ -2632,9 +2833,21 @@ array_subscr_lock_held(PyObject *op, PyObject *item) static PyObject * array_subscr(PyObject *op, PyObject *item) { + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i==-1 && PyErr_Occurred()) { + return NULL; + } + Py_ssize_t size = Pyarrayobject_GET_SIZE(op); + if (i < 0) { + i += size; + } + return array_item(op, i); + } + PyObject *ret; Py_BEGIN_CRITICAL_SECTION(op); - ret = array_subscr_lock_held(op, item); + ret = array_subscr_slice_lock_held(op, item); Py_END_CRITICAL_SECTION(); return ret; } @@ -2674,7 +2887,7 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) slicelength = 1; } else - return (*self->ob_descr->setitem)(self, i, value); + return (*self->ob_descr->setitem)(self->ob_item, i, value); } else if (PySlice_Check(item)) { if (PySlice_Unpack(item, &start, &stop, &step) < 0) { @@ -2808,17 +3021,28 @@ static int array_ass_subscr(PyObject *op, PyObject* item, PyObject* value) { int ret; - array_state* state = find_array_state_by_type(Py_TYPE(op)); - if (value != NULL && array_Check(value, state)) { - Py_BEGIN_CRITICAL_SECTION2(op, value); - ret = array_ass_subscr_lock_held(op, item, value); - Py_END_CRITICAL_SECTION2(); - } - else { - Py_BEGIN_CRITICAL_SECTION(op); - ret = array_ass_subscr_lock_held(op, item, value); - Py_END_CRITICAL_SECTION(); + + if (value != NULL) { + array_state* state = find_array_state_by_type(Py_TYPE(op)); + if (array_Check(value, state)) { + Py_BEGIN_CRITICAL_SECTION2(op, value); + ret = array_ass_subscr_lock_held(op, item, value); + Py_END_CRITICAL_SECTION2(); + return ret; + } + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return -1; + if (i < 0) + i += Pyarrayobject_GET_SIZE(op); + return setarrayitem_maybe_locked(op, i, value); + } } + + Py_BEGIN_CRITICAL_SECTION(op); + ret = array_ass_subscr_lock_held(op, item, value); + Py_END_CRITICAL_SECTION(); return ret; } @@ -2937,6 +3161,7 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) if (len > 0 && !array_Check(initial, state)) { Py_ssize_t i; + arrayobject *ap = arrayobject_CAST(a); for (i = 0; i < len; i++) { PyObject *v = PySequence_GetItem(initial, i); @@ -2944,7 +3169,7 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) Py_DECREF(a); return NULL; } - if (setarrayitem(a, i, v) != 0) { + if (setarrayitem(a, i, v, ap->ob_item) != 0) { Py_DECREF(v); Py_DECREF(a); return NULL; @@ -2973,9 +3198,20 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) if (n > 0) { arrayobject *self = (arrayobject *)a; +#ifdef Py_GIL_DISABLED + self->ob_item = arrayarray_set(self->ob_item, + (char *)ustr, n, sizeof(wchar_t), false); + PyMem_Free(ustr); + if (self->ob_item == NULL) { + Py_DECREF(a); + PyErr_NoMemory(); + return NULL; + } +#else // self->ob_item may be NULL but it is safe. PyMem_Free(self->ob_item); self->ob_item = (char *)ustr; +#endif Py_SET_SIZE(self, n); self->allocated = n; } @@ -2989,9 +3225,20 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) } arrayobject *self = (arrayobject *)a; +#ifdef Py_GIL_DISABLED + self->ob_item = arrayarray_set(self->ob_item, + (char *)ustr, n, sizeof(Py_UCS4), false); + PyMem_Free(ustr); + if (self->ob_item == NULL) { + Py_DECREF(a); + PyErr_NoMemory(); + return NULL; + } +#else // self->ob_item may be NULL but it is safe. PyMem_Free(self->ob_item); self->ob_item = (char *)ustr; +#endif Py_SET_SIZE(self, n); self->allocated = n; } @@ -3223,7 +3470,6 @@ array_iter(PyObject *op) it->ao = (arrayobject*)Py_NewRef(ao); it->index = 0; // -1 indicates exhausted - it->getitem = ao->ob_descr->getitem; PyObject_GC_Track(it); return (PyObject *)it; } @@ -3237,23 +3483,13 @@ arrayiter_next(PyObject *op) if (index < 0) { return NULL; } - PyObject *ret; arrayobject *ao = it->ao; #ifndef NDEBUG array_state *state = find_array_state_by_type(Py_TYPE(it)); assert(PyObject_TypeCheck(it, state->ArrayIterType)); assert(array_Check(ao, state)); #endif - - Py_BEGIN_CRITICAL_SECTION(ao); - if (index < Py_SIZE(ao)) { - ret = (*it->getitem)(ao, index); - } - else { - ret = NULL; - } - Py_END_CRITICAL_SECTION(); - + PyObject *ret = getarrayitem_maybe_locked((PyObject *)ao, index); if (ret != NULL) { FT_ATOMIC_STORE_SSIZE_RELAXED(it->index, index + 1); } From 512c4c74c5022eae0c416b992c56a63a805c2a05 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sun, 2 Mar 2025 21:08:06 +0000 Subject: [PATCH 03/32] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2025-03-02-21-08-05.gh-issue-128942.4MTI6s.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2025-03-02-21-08-05.gh-issue-128942.4MTI6s.rst diff --git a/Misc/NEWS.d/next/Library/2025-03-02-21-08-05.gh-issue-128942.4MTI6s.rst b/Misc/NEWS.d/next/Library/2025-03-02-21-08-05.gh-issue-128942.4MTI6s.rst new file mode 100644 index 00000000000000..0f2eb614e42374 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-02-21-08-05.gh-issue-128942.4MTI6s.rst @@ -0,0 +1 @@ +Make :mod:`array` module safe under :term:`free threading`. From 060100fb125c836643ac7c7a3432461184d50266 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 2 Mar 2025 16:36:24 -0500 Subject: [PATCH 04/32] ensure_shared_on_resize() in one more place --- Modules/arraymodule.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 41a5371acaf125..3ea1132a9f5aa4 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -250,6 +250,7 @@ array_resize(arrayobject *self, Py_ssize_t newsize) if (newsize == 0) { #ifdef Py_GIL_DISABLED + ensure_shared_on_resize(self); arrayarray_free(self->ob_item, _PyObject_GC_IS_SHARED(self)); #else arrayarray_free(self->ob_item, false); @@ -799,7 +800,6 @@ getarrayitem(PyObject *op, Py_ssize_t i, char *items) assert(array_Check(op, state)); #endif arrayobject *ap = arrayobject_CAST(op); - assert(valid_index(i, Py_SIZE(op))); return (*ap->ob_descr->getitem)(items, i); } @@ -864,7 +864,6 @@ setarrayitem(PyObject *op, Py_ssize_t i, PyObject *v, char *items) assert(array_Check(op, state)); #endif arrayobject *ap = arrayobject_CAST(op); - assert(valid_index(i, Py_SIZE(op))); return (*ap->ob_descr->setitem)(items, i, v); } From d0b17c6868f8e270c3690ac47c72a3e924f8edbc Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 3 Mar 2025 07:04:41 -0500 Subject: [PATCH 05/32] fix stupid direct return out of critical section --- Modules/arraymodule.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 3ea1132a9f5aa4..592d0ce784a7e4 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -814,10 +814,12 @@ getarrayitem_locked(PyObject *op, Py_ssize_t i) } #endif if (!valid_index(i, Py_SIZE(op))) { - return NULL; + ret = NULL; + } + else { + arrayobject *ap = (arrayobject *)op; + ret = getarrayitem(op, i, ap->ob_item); } - arrayobject *ap = (arrayobject *)op; - ret = getarrayitem(op, i, ap->ob_item); Py_END_CRITICAL_SECTION(); return ret; } @@ -844,6 +846,8 @@ getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) return NULL; } return getarrayitem(op, i, items); + /* Could check size again here to make sure it hasn't changed during get, + but not sure would add anything of value. */ } #else // Py_GIL_DISABLED From c17b787dc6871a5d17cfef3a30e12a4de9848875 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 3 Mar 2025 08:08:33 -0500 Subject: [PATCH 06/32] requested changes --- Modules/arraymodule.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 592d0ce784a7e4..67f8ca9229e84b 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -69,7 +69,7 @@ typedef struct { PyObject *str_iter; } array_state; -static inline Py_ssize_t Pyarrayobject_GET_SIZE(PyObject *op) { +static inline Py_ssize_t PyArray_GET_SIZE(PyObject *op) { arrayobject *ao = (arrayobject *)op; #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&(_PyVarObject_CAST(ao)->ob_size)); @@ -77,7 +77,7 @@ static inline Py_ssize_t Pyarrayobject_GET_SIZE(PyObject *op) { return Py_SIZE(ao); #endif } -#define Pyarrayobject_GET_SIZE(op) Pyarrayobject_GET_SIZE(_PyObject_CAST(op)) +#define PyArray_GET_SIZE(op) PyArray_GET_SIZE(_PyObject_CAST(op)) /* Forward declaration. */ static PyObject *array_array_frombytes(PyObject *self, PyObject *bytes); @@ -832,7 +832,7 @@ getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) if (!_Py_IsOwnedByCurrentThread((PyObject *)op) && !_PyObject_GC_IS_SHARED(op)) { return getarrayitem_locked(op, i); } - Py_ssize_t size = Pyarrayobject_GET_SIZE(op); + Py_ssize_t size = PyArray_GET_SIZE(op); if (!valid_index(i, size)) { return NULL; } @@ -901,12 +901,12 @@ setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) if (!_Py_IsOwnedByCurrentThread((PyObject *)op) && !_PyObject_GC_IS_SHARED(op)) { return setarrayitem_locked(op, i, v); } - Py_ssize_t size = Pyarrayobject_GET_SIZE(op); + Py_ssize_t size = PyArray_GET_SIZE(op); if (!valid_index(i, size)) { goto error; } arrayobject *ap = (arrayobject *)op; - char *items = _Py_atomic_load_ptr(&ap->ob_item); + char *items = _Py_atomic_load_ptr_relaxed(&ap->ob_item); if (items == NULL) { goto error; } @@ -981,7 +981,7 @@ array_dealloc(PyObject *op) if (self->ob_exports > 0) { PyErr_SetString(PyExc_SystemError, "deallocated array object has exported buffers"); - PyErr_Print(); + PyErr_WriteUnraisable(NULL); } if (self->weakreflist != NULL) { PyObject_ClearWeakRefs(op); @@ -1119,7 +1119,7 @@ static Py_ssize_t array_length(PyObject *op) { arrayobject *self = arrayobject_CAST(op); - return Pyarrayobject_GET_SIZE(self); + return PyArray_GET_SIZE(self); } @@ -2841,7 +2841,7 @@ array_subscr(PyObject *op, PyObject *item) if (i==-1 && PyErr_Occurred()) { return NULL; } - Py_ssize_t size = Pyarrayobject_GET_SIZE(op); + Py_ssize_t size = PyArray_GET_SIZE(op); if (i < 0) { i += size; } @@ -3038,7 +3038,7 @@ array_ass_subscr(PyObject *op, PyObject* item, PyObject* value) if (i == -1 && PyErr_Occurred()) return -1; if (i < 0) - i += Pyarrayobject_GET_SIZE(op); + i += PyArray_GET_SIZE(op); return setarrayitem_maybe_locked(op, i, value); } } @@ -3570,7 +3570,7 @@ array_arrayiterator___setstate__(arrayiterobject *self, PyObject *state) index = -1; } else { - Py_ssize_t size = Pyarrayobject_GET_SIZE(self->ao); + Py_ssize_t size = PyArray_GET_SIZE(self->ao); if (index > size) { index = size; /* iterator at end */ } From 4fd8383a558f316b62f3eb2a4009b5f0569371cf Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 3 Mar 2025 10:24:42 -0500 Subject: [PATCH 07/32] downgrade to _Py_atomic_load_ptr_relaxed in missed place --- Modules/arraymodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 67f8ca9229e84b..38967e2ed1bcd7 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -837,7 +837,7 @@ getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) return NULL; } arrayobject *ap = (arrayobject *)op; - char *items = _Py_atomic_load_ptr(&ap->ob_item); + char *items = _Py_atomic_load_ptr_relaxed(&ap->ob_item); if (items == NULL) { return NULL; } From d00f2b754a04def598977f7bfc5299ab436d1f4c Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Tue, 4 Mar 2025 13:46:11 -0500 Subject: [PATCH 08/32] arraymodule linked statically --- Include/internal/pycore_pymem.h | 2 +- Modules/Setup.bootstrap.in | 3 +++ Modules/Setup.stdlib.in | 1 - 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index bcd13d80cba975..5386d4c5f83031 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -117,7 +117,7 @@ extern wchar_t *_PyMem_DefaultRawWcsdup(const wchar_t *str); extern int _PyMem_DebugEnabled(void); // Enqueue a pointer to be freed possibly after some delay. -PyAPI_FUNC(void) _PyMem_FreeDelayed(void *ptr); +extern void _PyMem_FreeDelayed(void *ptr); // Enqueue an object to be freed possibly after some delay #ifdef Py_GIL_DISABLED diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 4dcc0f55176d0e..8b9f3465fddb2e 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -37,3 +37,6 @@ _symtable symtablemodule.c # for systems without $HOME env, used by site._getuserbase() @MODULE_PWD_TRUE@pwd pwdmodule.c + +# for optimization purposes +@MODULE_ARRAY_TRUE@array arraymodule.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 6bb05a06a3465d..17fa05703fd6f1 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -28,7 +28,6 @@ ############################################################################ # Modules that should always be present (POSIX and Windows): -@MODULE_ARRAY_TRUE@array arraymodule.c @MODULE__ASYNCIO_TRUE@_asyncio _asynciomodule.c @MODULE__BISECT_TRUE@_bisect _bisectmodule.c @MODULE__CONTEXTVARS_TRUE@_contextvars _contextvarsmodule.c From a3e60046124613ce76f1f9c270e7af2346134217 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Wed, 5 Mar 2025 12:07:39 -0500 Subject: [PATCH 09/32] cleanups --- Lib/test/test_array.py | 4 +- Modules/arraymodule.c | 490 ++++++++++++++++++++--------------------- 2 files changed, 243 insertions(+), 251 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index a7d2105b938598..41b52ea424faff 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1172,14 +1172,14 @@ def test_create_from_bytes(self): @support.cpython_only def test_sizeof_with_buffer(self): a = array.array(self.typecode, self.example) - basesize = support.calcvobjsize('Pn2Pi') + basesize = support.calcvobjsize('PnPi') buffer_size = a.buffer_info()[1] * a.itemsize support.check_sizeof(self, a, basesize + buffer_size) @support.cpython_only def test_sizeof_without_buffer(self): a = array.array(self.typecode) - basesize = support.calcvobjsize('Pn2Pi') + basesize = support.calcvobjsize('PnPi') support.check_sizeof(self, a, basesize) def test_initialize_with_unicode(self): diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 38967e2ed1bcd7..d24bba34a0312c 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -42,10 +42,14 @@ struct arraydescr { int is_signed; }; +typedef struct { + Py_ssize_t allocated; + char items[]; +} arraydata; + typedef struct arrayobject { PyObject_VAR_HEAD - char *ob_item; - Py_ssize_t allocated; + arraydata *data; const struct arraydescr *ob_descr; PyObject *weakreflist; /* List of weak references */ Py_ssize_t ob_exports; /* Number of exported buffers */ @@ -69,7 +73,8 @@ typedef struct { PyObject *str_iter; } array_state; -static inline Py_ssize_t PyArray_GET_SIZE(PyObject *op) { +static inline +Py_ssize_t PyArray_GET_SIZE(PyObject *op) { arrayobject *ao = (arrayobject *)op; #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&(_PyVarObject_CAST(ao)->ob_size)); @@ -144,91 +149,74 @@ enum machine_format_code { #define array_Check(op, state) PyObject_TypeCheck(op, state->ArrayType) -#ifdef Py_GIL_DISABLED -typedef struct { - Py_ssize_t allocated; - char ob_item[]; -} _PyArrayArray; -#endif +static inline char * +arraydata_safeitems(arraydata *data) +{ + return data == NULL ? NULL : data->items; +} -static char * -arrayarray_alloc(Py_ssize_t size, int itemsize) +static arraydata * +arraydata_alloc(Py_ssize_t size, int itemsize) { -#ifdef Py_GIL_DISABLED if (size > PY_SSIZE_T_MAX/itemsize - 1) { return NULL; } - _PyArrayArray *array = PyMem_Malloc(sizeof(_PyArrayArray) + size * itemsize); - if (array == NULL) { + arraydata *data = (arraydata *)PyMem_Malloc(sizeof(arraydata) + size * itemsize); + if (data == NULL) { return NULL; } - array->allocated = size; - return array->ob_item; -#else - return PyMem_Malloc(size * itemsize); -#endif + data->allocated = size; + return data; } static void -arrayarray_free(char *items, bool use_qsbr) +arraydata_free(arraydata *data, bool use_qsbr) { #ifdef Py_GIL_DISABLED - if (items == NULL) { - return; - } - _PyArrayArray *array = _Py_CONTAINER_OF(items, _PyArrayArray, ob_item); if (use_qsbr) { - _PyMem_FreeDelayed(array); + if (data != NULL) { + _PyMem_FreeDelayed(data); + } } else { - PyMem_Free(array); + PyMem_Free(data); } #else - PyMem_Free(items); + PyMem_Free(data); #endif } -#ifdef Py_GIL_DISABLED - -static Py_ssize_t -arrayarray_capacity(char *items) +static arraydata * +arraydata_set_items(arraydata *data, char *newitems, Py_ssize_t newsize, int itemsize, bool use_qsbr) { - _PyArrayArray *array = _Py_CONTAINER_OF(items, _PyArrayArray, ob_item); - return array->allocated; -} - -static char * -arrayarray_set(char *items, char *newitems, Py_ssize_t newsize, int itemsize, bool use_qsbr) -{ - arrayarray_free(items, use_qsbr); - items = arrayarray_alloc(newsize, itemsize); - if (items != NULL) { - memcpy(items, newitems, newsize * itemsize); + arraydata_free(data, use_qsbr); + data = arraydata_alloc(newsize, itemsize); + if (data != NULL) { + memcpy(data->items, newitems, newsize * itemsize); } - return items; + return data; } -static void -ensure_shared_on_resize(arrayobject *self) +#ifndef Py_GIL_DISABLED +static arraydata * +arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) { - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); - // Ensure that the array is freed using QSBR if we are not the - // owning thread. - if (!_Py_IsOwnedByCurrentThread((PyObject *)self) && - !_PyObject_GC_IS_SHARED(self)) - { - _PyObject_GC_SET_SHARED(self); + if (size > PY_SSIZE_T_MAX/itemsize - 1) { + return NULL; + } + data = (arraydata *)PyMem_Realloc(data, sizeof(arraydata) + size * itemsize); + if (data == NULL) { + return NULL; } + data->allocated = size; + return data; } - -# endif // Py_GIL_DISABLED +#endif static int array_resize(arrayobject *self, Py_ssize_t newsize) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); - char *items; - size_t _new_size; if (self->ob_exports > 0 && newsize != Py_SIZE(self)) { PyErr_SetString(PyExc_BufferError, @@ -241,23 +229,33 @@ array_resize(arrayobject *self, Py_ssize_t newsize) current size, then proceed with the realloc() to shrink the array. */ - if (self->allocated >= newsize && - Py_SIZE(self) < newsize + 16 && - self->ob_item != NULL) { + arraydata *data = self->data; + + if (data != NULL && + data->allocated >= newsize && + Py_SIZE(self) < newsize + 16) { Py_SET_SIZE(self, newsize); return 0; } +#ifdef Py_GIL_DISABLED + // Ensure that the array is freed using QSBR if we are not the + // owning thread. + if (!_Py_IsOwnedByCurrentThread((PyObject *)self) && + !_PyObject_GC_IS_SHARED(self)) + { + _PyObject_GC_SET_SHARED(self); + } +#endif + if (newsize == 0) { #ifdef Py_GIL_DISABLED - ensure_shared_on_resize(self); - arrayarray_free(self->ob_item, _PyObject_GC_IS_SHARED(self)); + arraydata_free(self->data, _PyObject_GC_IS_SHARED(self)); #else - arrayarray_free(self->ob_item, false); + arraydata_free(self->data, false); #endif - self->ob_item = NULL; + self->data = NULL; Py_SET_SIZE(self, 0); - FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, 0); return 0; } @@ -273,8 +271,7 @@ array_resize(arrayobject *self, Py_ssize_t newsize) * memory critical. */ - _new_size = (newsize >> 4) + (Py_SIZE(self) < 8 ? 3 : 7) + newsize; - items = self->ob_item; + size_t _new_size = (newsize >> 4) + (Py_SIZE(self) < 8 ? 3 : 7) + newsize; int itemsize = self->ob_descr->itemsize; /* XXX The following multiplication and division does not optimize away @@ -285,29 +282,27 @@ array_resize(arrayobject *self, Py_ssize_t newsize) } #ifdef Py_GIL_DISABLED - ensure_shared_on_resize(self); - char *newitems = arrayarray_alloc(_new_size, itemsize); - if (newitems == NULL) { + arraydata *newdata = arraydata_alloc(_new_size, itemsize); + if (newdata == NULL) { PyErr_NoMemory(); return -1; } - if (items != NULL) { + if (data != NULL) { Py_ssize_t size = Py_SIZE(self); - memcpy(newitems, items, Py_MIN(size, newsize) * itemsize); - arrayarray_free(items, _PyObject_GC_IS_SHARED(self)); + memcpy(newdata->items, data->items, Py_MIN(size, newsize) * itemsize); + arraydata_free(data, _PyObject_GC_IS_SHARED(self)); } - _Py_atomic_store_ptr_release(&self->ob_item, newitems); + _Py_atomic_store_ptr_release(&self->data, newdata); #else - PyMem_RESIZE(items, char, (_new_size * itemsize)); - if (items == NULL) { + data = arraydata_realloc(data, _new_size, itemsize); + if (data == NULL) { PyErr_NoMemory(); return -1; } - self->ob_item = items; + self->data = data; #endif Py_SET_SIZE(self, newsize); - FT_ATOMIC_STORE_SSIZE_RELAXED(self->allocated, _new_size); return 0; } @@ -324,14 +319,14 @@ in bounds; that's the responsibility of the caller. ****************************************************************************/ static PyObject * -b_getitem(char *ob_item, Py_ssize_t i) +b_getitem(char *items, Py_ssize_t i) { - long x = ((signed char *)ob_item)[i]; + long x = ((signed char *)items)[i]; return PyLong_FromLong(x); } static int -b_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +b_setitem(char *items, Py_ssize_t i, PyObject *v) { short x; /* PyArg_Parse's 'b' formatter is for an unsigned char, therefore @@ -350,37 +345,37 @@ b_setitem(char *ob_item, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((char *)ob_item)[i] = (char)x; + ((char *)items)[i] = (char)x; return 0; } static PyObject * -BB_getitem(char *ob_item, Py_ssize_t i) +BB_getitem(char *items, Py_ssize_t i) { - long x = ((unsigned char *)ob_item)[i]; + long x = ((unsigned char *)items)[i]; return PyLong_FromLong(x); } static int -BB_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +BB_setitem(char *items, Py_ssize_t i, PyObject *v) { unsigned char x; /* 'B' == unsigned char, maps to PyArg_Parse's 'b' formatter */ if (!PyArg_Parse(v, "b;array item must be integer", &x)) return -1; if (i >= 0) - ((unsigned char *)ob_item)[i] = x; + ((unsigned char *)items)[i] = x; return 0; } static PyObject * -u_getitem(char *ob_item, Py_ssize_t i) +u_getitem(char *items, Py_ssize_t i) { - return PyUnicode_FromOrdinal(((wchar_t *) ob_item)[i]); + return PyUnicode_FromOrdinal(((wchar_t *) items)[i]); } static int -u_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +u_setitem(char *items, Py_ssize_t i, PyObject *v) { if (!PyUnicode_Check(v)) { PyErr_Format(PyExc_TypeError, @@ -411,19 +406,19 @@ u_setitem(char *ob_item, Py_ssize_t i, PyObject *v) assert(len == 1); if (i >= 0) { - ((wchar_t *)ob_item)[i] = w; + ((wchar_t *)items)[i] = w; } return 0; } static PyObject * -w_getitem(char *ob_item, Py_ssize_t i) +w_getitem(char *items, Py_ssize_t i) { - return PyUnicode_FromOrdinal(((Py_UCS4 *) ob_item)[i]); + return PyUnicode_FromOrdinal(((Py_UCS4 *) items)[i]); } static int -w_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +w_setitem(char *items, Py_ssize_t i, PyObject *v) { if (!PyUnicode_Check(v)) { PyErr_Format(PyExc_TypeError, @@ -441,38 +436,38 @@ w_setitem(char *ob_item, Py_ssize_t i, PyObject *v) } if (i >= 0) { - ((Py_UCS4 *)ob_item)[i] = PyUnicode_READ_CHAR(v, 0); + ((Py_UCS4 *)items)[i] = PyUnicode_READ_CHAR(v, 0); } return 0; } static PyObject * -h_getitem(char *ob_item, Py_ssize_t i) +h_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong((long) ((short *)ob_item)[i]); + return PyLong_FromLong((long) ((short *)items)[i]); } static int -h_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +h_setitem(char *items, Py_ssize_t i, PyObject *v) { short x; /* 'h' == signed short, maps to PyArg_Parse's 'h' formatter */ if (!PyArg_Parse(v, "h;array item must be integer", &x)) return -1; if (i >= 0) - ((short *)ob_item)[i] = x; + ((short *)items)[i] = x; return 0; } static PyObject * -HH_getitem(char *ob_item, Py_ssize_t i) +HH_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong((long) ((unsigned short *)ob_item)[i]); + return PyLong_FromLong((long) ((unsigned short *)items)[i]); } static int -HH_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +HH_setitem(char *items, Py_ssize_t i, PyObject *v) { int x; /* PyArg_Parse's 'h' formatter is for a signed short, therefore @@ -490,37 +485,37 @@ HH_setitem(char *ob_item, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((short *)ob_item)[i] = (short)x; + ((short *)items)[i] = (short)x; return 0; } static PyObject * -i_getitem(char *ob_item, Py_ssize_t i) +i_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong((long) ((int *)ob_item)[i]); + return PyLong_FromLong((long) ((int *)items)[i]); } static int -i_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +i_setitem(char *items, Py_ssize_t i, PyObject *v) { int x; /* 'i' == signed int, maps to PyArg_Parse's 'i' formatter */ if (!PyArg_Parse(v, "i;array item must be integer", &x)) return -1; if (i >= 0) - ((int *)ob_item)[i] = x; + ((int *)items)[i] = x; return 0; } static PyObject * -II_getitem(char *ob_item, Py_ssize_t i) +II_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLong( - (unsigned long) ((unsigned int *)ob_item)[i]); + (unsigned long) ((unsigned int *)items)[i]); } static int -II_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +II_setitem(char *items, Py_ssize_t i, PyObject *v) { unsigned long x; int do_decref = 0; /* if nb_int was called */ @@ -548,7 +543,7 @@ II_setitem(char *ob_item, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((unsigned int *)ob_item)[i] = (unsigned int)x; + ((unsigned int *)items)[i] = (unsigned int)x; if (do_decref) { Py_DECREF(v); @@ -557,30 +552,30 @@ II_setitem(char *ob_item, Py_ssize_t i, PyObject *v) } static PyObject * -l_getitem(char *ob_item, Py_ssize_t i) +l_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong(((long *)ob_item)[i]); + return PyLong_FromLong(((long *)items)[i]); } static int -l_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +l_setitem(char *items, Py_ssize_t i, PyObject *v) { long x; if (!PyArg_Parse(v, "l;array item must be integer", &x)) return -1; if (i >= 0) - ((long *)ob_item)[i] = x; + ((long *)items)[i] = x; return 0; } static PyObject * -LL_getitem(char *ob_item, Py_ssize_t i) +LL_getitem(char *items, Py_ssize_t i) { - return PyLong_FromUnsignedLong(((unsigned long *)ob_item)[i]); + return PyLong_FromUnsignedLong(((unsigned long *)items)[i]); } static int -LL_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +LL_setitem(char *items, Py_ssize_t i, PyObject *v) { unsigned long x; int do_decref = 0; /* if nb_int was called */ @@ -600,7 +595,7 @@ LL_setitem(char *ob_item, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((unsigned long *)ob_item)[i] = x; + ((unsigned long *)items)[i] = x; if (do_decref) { Py_DECREF(v); @@ -609,31 +604,31 @@ LL_setitem(char *ob_item, Py_ssize_t i, PyObject *v) } static PyObject * -q_getitem(char *ob_item, Py_ssize_t i) +q_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLongLong(((long long *)ob_item)[i]); + return PyLong_FromLongLong(((long long *)items)[i]); } static int -q_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +q_setitem(char *items, Py_ssize_t i, PyObject *v) { long long x; if (!PyArg_Parse(v, "L;array item must be integer", &x)) return -1; if (i >= 0) - ((long long *)ob_item)[i] = x; + ((long long *)items)[i] = x; return 0; } static PyObject * -QQ_getitem(char *ob_item, Py_ssize_t i) +QQ_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLongLong( - ((unsigned long long *)ob_item)[i]); + ((unsigned long long *)items)[i]); } static int -QQ_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +QQ_setitem(char *items, Py_ssize_t i, PyObject *v) { unsigned long long x; int do_decref = 0; /* if nb_int was called */ @@ -653,7 +648,7 @@ QQ_setitem(char *ob_item, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) - ((unsigned long long *)ob_item)[i] = x; + ((unsigned long long *)items)[i] = x; if (do_decref) { Py_DECREF(v); @@ -662,36 +657,36 @@ QQ_setitem(char *ob_item, Py_ssize_t i, PyObject *v) } static PyObject * -f_getitem(char *ob_item, Py_ssize_t i) +f_getitem(char *items, Py_ssize_t i) { - return PyFloat_FromDouble((double) ((float *)ob_item)[i]); + return PyFloat_FromDouble((double) ((float *)items)[i]); } static int -f_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +f_setitem(char *items, Py_ssize_t i, PyObject *v) { float x; if (!PyArg_Parse(v, "f;array item must be float", &x)) return -1; if (i >= 0) - ((float *)ob_item)[i] = x; + ((float *)items)[i] = x; return 0; } static PyObject * -d_getitem(char *ob_item, Py_ssize_t i) +d_getitem(char *items, Py_ssize_t i) { - return PyFloat_FromDouble(((double *)ob_item)[i]); + return PyFloat_FromDouble(((double *)items)[i]); } static int -d_setitem(char *ob_item, Py_ssize_t i, PyObject *v) +d_setitem(char *items, Py_ssize_t i, PyObject *v) { double x; if (!PyArg_Parse(v, "d;array item must be float", &x)) return -1; if (i >= 0) - ((double *)ob_item)[i] = x; + ((double *)items)[i] = x; return 0; } @@ -769,15 +764,14 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *des return NULL; } op->ob_descr = descr; - op->allocated = size; op->weakreflist = NULL; Py_SET_SIZE(op, size); if (size <= 0) { - op->ob_item = NULL; + op->data = NULL; } else { - op->ob_item = arrayarray_alloc(size, descr->itemsize); - if (op->ob_item == NULL) { + op->data = arraydata_alloc(size, descr->itemsize); + if (op->data == NULL) { Py_DECREF(op); return PyErr_NoMemory(); } @@ -792,14 +786,18 @@ valid_index(Py_ssize_t i, Py_ssize_t limit) return (size_t) i < (size_t) limit; } -static PyObject * -getarrayitem(PyObject *op, Py_ssize_t i, char *items) +static inline PyObject * +getarrayitem(arrayobject *ap, Py_ssize_t i, char *items) { #ifndef NDEBUG - array_state *state = find_array_state_by_type(Py_TYPE(op)); - assert(array_Check(op, state)); + array_state *state = find_array_state_by_type(Py_TYPE(ap)); + assert(array_Check(ap, state)); +#ifdef Py_GIL_DISABLED + assert(valid_index(i, (_Py_CONTAINER_OF(items, arraydata, items))->allocated)); +#else + assert(valid_index(i, Py_SIZE(ap))); +#endif #endif - arrayobject *ap = arrayobject_CAST(op); return (*ap->ob_descr->getitem)(items, i); } @@ -818,7 +816,7 @@ getarrayitem_locked(PyObject *op, Py_ssize_t i) } else { arrayobject *ap = (arrayobject *)op; - ret = getarrayitem(op, i, ap->ob_item); + ret = getarrayitem(ap, i, ap->data->items); } Py_END_CRITICAL_SECTION(); return ret; @@ -837,17 +835,17 @@ getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) return NULL; } arrayobject *ap = (arrayobject *)op; - char *items = _Py_atomic_load_ptr_relaxed(&ap->ob_item); - if (items == NULL) { + arraydata *data = _Py_atomic_load_ptr_acquire(&ap->data); + if (data == NULL) { return NULL; } - Py_ssize_t cap = arrayarray_capacity(items); - if (!valid_index(i, cap)) { + if (!valid_index(i, data->allocated)) { return NULL; } - return getarrayitem(op, i, items); - /* Could check size again here to make sure it hasn't changed during get, - but not sure would add anything of value. */ + return getarrayitem(ap, i, data->items); + /* Could check size again here to make sure it hasn't changed but since + there isn't a well defined order between unsynchronized thread + operations there's no point. */ } #else // Py_GIL_DISABLED @@ -860,14 +858,20 @@ getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) #endif // Py_GIL_DISABLED -static int -setarrayitem(PyObject *op, Py_ssize_t i, PyObject *v, char *items) +static inline int +setarrayitem(arrayobject *ap, Py_ssize_t i, PyObject *v, char *items) { #ifndef NDEBUG - array_state *state = find_array_state_by_type(Py_TYPE(op)); - assert(array_Check(op, state)); + array_state *state = find_array_state_by_type(Py_TYPE(ap)); + assert(array_Check(ap, state)); + if (items != NULL) { +#ifdef Py_GIL_DISABLED + assert(valid_index(i, (_Py_CONTAINER_OF(items, arraydata, items))->allocated)); +#else + assert(valid_index(i, Py_SIZE(ap))); +#endif + } #endif - arrayobject *ap = arrayobject_CAST(op); return (*ap->ob_descr->setitem)(items, i, v); } @@ -886,8 +890,8 @@ setarrayitem_locked(PyObject *op, Py_ssize_t i, PyObject *v) ret = -1; } else { - arrayobject *ap = (arrayobject *)op; - ret = setarrayitem(op, i, v, ap->ob_item); + arrayobject *ap = arrayobject_CAST(op); + ret = setarrayitem(ap, i, v, ap->data->items); } Py_END_CRITICAL_SECTION(); return ret; @@ -906,15 +910,16 @@ setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) goto error; } arrayobject *ap = (arrayobject *)op; - char *items = _Py_atomic_load_ptr_relaxed(&ap->ob_item); - if (items == NULL) { + arraydata *data = _Py_atomic_load_ptr_relaxed(&ap->data); + if (data == NULL) { goto error; } - Py_ssize_t cap = arrayarray_capacity(items); - if (!valid_index(i, cap)) { + if (!valid_index(i, data->allocated)) { goto error; } - return setarrayitem(op, i, v, items); + int ret = setarrayitem(ap, i, v, data->items); + _Py_atomic_fence_release(); + return ret; error: PyErr_SetString(PyExc_IndexError, "array index out of range"); @@ -941,12 +946,12 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) PyErr_BadInternalCall(); return -1; } - if ((*self->ob_descr->setitem)(self->ob_item, -1, v) < 0) + if (setarrayitem(self, -1, v, NULL) < 0) return -1; if (array_resize(self, n+1) == -1) return -1; - items = self->ob_item; + items = self->data->items; if (where < 0) { where += n; if (where < 0) @@ -959,7 +964,7 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) memmove(items + (where+1)*self->ob_descr->itemsize, items + where*self->ob_descr->itemsize, (n-where)*self->ob_descr->itemsize); - return (*self->ob_descr->setitem)(self->ob_item, where, v); + return setarrayitem(self, where, v, items); } /* Methods */ @@ -986,8 +991,8 @@ array_dealloc(PyObject *op) if (self->weakreflist != NULL) { PyObject_ClearWeakRefs(op); } - if (self->ob_item != NULL) { - arrayarray_free(self->ob_item, false); + if (self->data != NULL) { + arraydata_free(self->data, false); } tp->tp_free(op); Py_DECREF(tp); @@ -1024,7 +1029,7 @@ array_richcompare_lock_held(PyObject *v, PyObject *w, int op) /* Fast path: arrays with same types can have their buffers compared directly */ Py_ssize_t common_length = Py_MIN(Py_SIZE(va), Py_SIZE(wa)); - int result = va->ob_descr->compareitems(va->ob_item, wa->ob_item, + int result = va->ob_descr->compareitems(va->data->items, wa->data->items, common_length); if (result == 0) goto compare_sizes; @@ -1047,11 +1052,11 @@ array_richcompare_lock_held(PyObject *v, PyObject *w, int op) /* Search for the first index where items are different */ k = 1; for (i = 0; i < Py_SIZE(va) && i < Py_SIZE(wa); i++) { - vi = getarrayitem(v, i, va->ob_item); + vi = getarrayitem(va, i, va->data->items); if (vi == NULL) { return NULL; } - wi = getarrayitem(w, i, wa->ob_item); + wi = getarrayitem(wa, i, wa->data->items); if (wi == NULL) { Py_DECREF(vi); return NULL; @@ -1155,7 +1160,7 @@ array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) if (np == NULL) return NULL; if (ihigh > ilow) { - memcpy(np->ob_item, a->ob_item + ilow * a->ob_descr->itemsize, + memcpy(np->data->items, a->data->items + ilow * a->ob_descr->itemsize, (ihigh-ilow) * a->ob_descr->itemsize); } return (PyObject *)np; @@ -1238,11 +1243,11 @@ array_concat_lock_held(PyObject *op, PyObject *bb) return NULL; } if (Py_SIZE(a) > 0) { - memcpy(np->ob_item, a->ob_item, Py_SIZE(a)*a->ob_descr->itemsize); + memcpy(np->data->items, a->data->items, Py_SIZE(a)*a->ob_descr->itemsize); } if (Py_SIZE(b) > 0) { - memcpy(np->ob_item + Py_SIZE(a)*a->ob_descr->itemsize, - b->ob_item, Py_SIZE(b)*b->ob_descr->itemsize); + memcpy(np->data->items + Py_SIZE(a)*a->ob_descr->itemsize, + b->data->items, Py_SIZE(b)*b->ob_descr->itemsize); } return (PyObject *)np; #undef b @@ -1280,7 +1285,7 @@ array_repeat_lock_held(PyObject *op, Py_ssize_t n) const Py_ssize_t oldbytes = array_length * a->ob_descr->itemsize; const Py_ssize_t newbytes = oldbytes * n; - _PyBytes_Repeat(np->ob_item, newbytes, a->ob_item, oldbytes); + _PyBytes_Repeat(np->data->items, newbytes, a->data->items, oldbytes); return (PyObject *)np; } @@ -1311,7 +1316,7 @@ array_del_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) ihigh = ilow; else if (ihigh > Py_SIZE(a)) ihigh = Py_SIZE(a); - item = a->ob_item; + item = a->data->items; d = ihigh-ilow; /* Issue #4509: If the array has exported buffers and the slice assignment would change the size of the array, fail early to make @@ -1396,8 +1401,8 @@ array_do_extend_lock_held(array_state *state, arrayobject *self, PyObject *bb) if (array_resize(self, size) == -1) return -1; if (bbsize > 0) { - memcpy(self->ob_item + oldsize * self->ob_descr->itemsize, - b->ob_item, bbsize * b->ob_descr->itemsize); + memcpy(self->data->items + oldsize * self->ob_descr->itemsize, + b->data->items, bbsize * b->ob_descr->itemsize); } return 0; @@ -1452,7 +1457,7 @@ array_inplace_repeat_lock_held(PyObject *op, Py_ssize_t n) if (array_resize(self, n * array_size) == -1) return NULL; - _PyBytes_Repeat(self->ob_item, n*size, self->ob_item, size); + _PyBytes_Repeat(self->data->items, n*size, self->data->items, size); } return Py_NewRef(self); } @@ -1489,7 +1494,7 @@ array_array_count_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem((PyObject *)self, i, self->ob_item); + selfi = getarrayitem(self, i, self->data->items); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1537,7 +1542,7 @@ array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, PyObject *selfi; int cmp; - selfi = getarrayitem((PyObject *)self, i, self->ob_item); + selfi = getarrayitem(self, i, self->data->items); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1561,7 +1566,7 @@ array_contains_lock_held(PyObject *op, PyObject *v) for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(op); i++) { arrayobject *ap = (arrayobject *)op; - PyObject *opi = getarrayitem(op, i, ap->ob_item); + PyObject *opi = getarrayitem(ap, i, ap->data->items); if (opi == NULL) return -1; cmp = PyObject_RichCompareBool(opi, v, Py_EQ); @@ -1600,7 +1605,7 @@ array_array_remove_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem((PyObject *)self, i, self->ob_item); + selfi = getarrayitem(self, i, self->data->items); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1646,7 +1651,7 @@ array_array_pop_impl(arrayobject *self, Py_ssize_t i) PyErr_SetString(PyExc_IndexError, "pop index out of range"); return NULL; } - v = getarrayitem((PyObject *)self, i, self->ob_item); + v = getarrayitem(self, i, self->data->items); if (v == NULL) return NULL; if (array_del_slice(self, i, i+1) != 0) { @@ -1717,7 +1722,7 @@ array_array_buffer_info_impl(arrayobject *self) if (!retval) return NULL; - v = PyLong_FromVoidPtr(self->ob_item); + v = PyLong_FromVoidPtr(arraydata_safeitems(self->data)); if (v == NULL) { Py_DECREF(retval); return NULL; @@ -1774,14 +1779,14 @@ array_array_byteswap_impl(arrayobject *self) case 1: break; case 2: - for (p = self->ob_item, i = Py_SIZE(self); --i >= 0; p += 2) { + for (p = self->data->items, i = Py_SIZE(self); --i >= 0; p += 2) { char p0 = p[0]; p[0] = p[1]; p[1] = p0; } break; case 4: - for (p = self->ob_item, i = Py_SIZE(self); --i >= 0; p += 4) { + for (p = self->data->items, i = Py_SIZE(self); --i >= 0; p += 4) { char p0 = p[0]; char p1 = p[1]; p[0] = p[3]; @@ -1791,7 +1796,7 @@ array_array_byteswap_impl(arrayobject *self) } break; case 8: - for (p = self->ob_item, i = Py_SIZE(self); --i >= 0; p += 8) { + for (p = self->data->items, i = Py_SIZE(self); --i >= 0; p += 8) { char p0 = p[0]; char p1 = p[1]; char p2 = p[2]; @@ -1832,8 +1837,8 @@ array_array_reverse_impl(arrayobject *self) assert((size_t)itemsize <= sizeof(tmp)); if (Py_SIZE(self) > 1) { - for (p = self->ob_item, - q = self->ob_item + (Py_SIZE(self) - 1)*itemsize; + for (p = self->data->items, + q = self->data->items + (Py_SIZE(self) - 1)*itemsize; p < q; p += itemsize, q -= itemsize) { /* memory areas guaranteed disjoint, so memcpy @@ -1942,7 +1947,7 @@ array_array_tofile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f) assert(state != NULL); for (i = 0; i < nblocks; i++) { - char* ptr = self->ob_item + i*BLOCKSIZE; + char *ptr = self->data->items + i*BLOCKSIZE; Py_ssize_t size = BLOCKSIZE; PyObject *bytes, *res; @@ -1990,8 +1995,7 @@ array_array_fromlist_impl(arrayobject *self, PyObject *list) return NULL; for (i = 0; i < n; i++) { PyObject *v = PyList_GET_ITEM(list, i); - if ((*self->ob_descr->setitem)(self->ob_item, - Py_SIZE(self) - n + i, v) != 0) { + if (setarrayitem(self, Py_SIZE(self) - n + i, v, self->data->items) != 0) { array_resize(self, old_size); return NULL; } @@ -2023,7 +2027,7 @@ array_array_tolist_impl(arrayobject *self) if (list == NULL) return NULL; for (i = 0; i < Py_SIZE(self); i++) { - PyObject *v = getarrayitem((PyObject *)self, i, self->ob_item); + PyObject *v = getarrayitem(self, i, self->data->items); if (v == NULL) goto error; PyList_SET_ITEM(list, i, v); @@ -2072,7 +2076,7 @@ array_array_frombytes_impl(arrayobject *self, Py_buffer *buffer) if (array_resize(self, old_size + n) == -1) { return NULL; } - memcpy(self->ob_item + old_size * itemsize, + memcpy(self->data->items + old_size * itemsize, buffer->buf, n * itemsize); } Py_RETURN_NONE; @@ -2090,7 +2094,7 @@ array_array_tobytes_impl(arrayobject *self) /*[clinic end generated code: output=87318e4edcdc2bb6 input=c4d44d5499d2320f]*/ { if (Py_SIZE(self) <= PY_SSIZE_T_MAX / self->ob_descr->itemsize) { - return PyBytes_FromStringAndSize(self->ob_item, + return PyBytes_FromStringAndSize(arraydata_safeitems(self->data), Py_SIZE(self) * self->ob_descr->itemsize); } else { return PyErr_NoMemory(); @@ -2135,7 +2139,7 @@ array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) // must not fail PyUnicode_AsWideChar( - ustr, ((wchar_t *)self->ob_item) + old_size, ustr_length); + ustr, ((wchar_t *)arraydata_safeitems(self->data)) + old_size, ustr_length); } } else { // typecode == 'w' @@ -2151,7 +2155,7 @@ array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) } // must not fail - Py_UCS4 *u = PyUnicode_AsUCS4(ustr, ((Py_UCS4*)self->ob_item) + old_size, + Py_UCS4 *u = PyUnicode_AsUCS4(ustr, ((Py_UCS4*)arraydata_safeitems(self->data)) + old_size, ustr_length, 0); assert(u != NULL); (void)u; // Suppress unused_variable warning. @@ -2182,11 +2186,11 @@ array_array_tounicode_impl(arrayobject *self) return NULL; } if (typecode == 'u') { - return PyUnicode_FromWideChar((wchar_t *) self->ob_item, Py_SIZE(self)); + return PyUnicode_FromWideChar((wchar_t *) arraydata_safeitems(self->data), Py_SIZE(self)); } else { // typecode == 'w' int byteorder = 0; // native byteorder - return PyUnicode_DecodeUTF32((const char *) self->ob_item, Py_SIZE(self) * 4, + return PyUnicode_DecodeUTF32((const char *) arraydata_safeitems(self->data), Py_SIZE(self) * 4, NULL, &byteorder); } } @@ -2202,8 +2206,10 @@ array_array___sizeof___impl(arrayobject *self) /*[clinic end generated code: output=d8e1c61ebbe3eaed input=805586565bf2b3c6]*/ { size_t res = _PyObject_SIZE(Py_TYPE(self)); - res += (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(self->allocated) - * (size_t)self->ob_descr->itemsize; + arraydata *data = FT_ATOMIC_LOAD_PTR_RELAXED(self->data); + if (data != NULL) { + res += (size_t)data->allocated * (size_t)self->ob_descr->itemsize; + } return PyLong_FromSize_t(res); } @@ -2805,8 +2811,8 @@ array_subscr_slice_lock_held(PyObject *op, PyObject *item) slicelength, self->ob_descr); if (result == NULL) return NULL; - memcpy(((arrayobject *)result)->ob_item, - self->ob_item + start * itemsize, + memcpy(((arrayobject *)result)->data->items, + self->data->items + start * itemsize, slicelength * itemsize); return result; } @@ -2818,8 +2824,8 @@ array_subscr_slice_lock_held(PyObject *op, PyObject *item) for (cur = start, i = 0; i < slicelength; cur += step, i++) { - memcpy(ar->ob_item + i*itemsize, - self->ob_item + cur*itemsize, + memcpy(ar->data->items + i*itemsize, + self->data->items + cur*itemsize, itemsize); } @@ -2890,7 +2896,7 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) slicelength = 1; } else - return (*self->ob_descr->setitem)(self->ob_item, i, value); + return setarrayitem(self, i, value, self->data->items); } else if (PySlice_Check(item)) { if (PySlice_Unpack(item, &start, &stop, &step) < 0) { @@ -2949,8 +2955,8 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) if (step == 1) { if (slicelength > needed) { - memmove(self->ob_item + (start + needed) * itemsize, - self->ob_item + stop * itemsize, + memmove(self->data->items + (start + needed) * itemsize, + self->data->items + stop * itemsize, (Py_SIZE(self) - stop) * itemsize); if (array_resize(self, Py_SIZE(self) + needed - slicelength) < 0) @@ -2960,13 +2966,13 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) if (array_resize(self, Py_SIZE(self) + needed - slicelength) < 0) return -1; - memmove(self->ob_item + (start + needed) * itemsize, - self->ob_item + stop * itemsize, + memmove(self->data->items + (start + needed) * itemsize, + self->data->items + stop * itemsize, (Py_SIZE(self) - start - needed) * itemsize); } if (needed > 0) - memcpy(self->ob_item + start * itemsize, - other->ob_item, needed * itemsize); + memcpy(self->data->items + start * itemsize, + other->data->items, needed * itemsize); return 0; } else if (needed == 0) { @@ -2985,14 +2991,14 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) if (cur + step >= (size_t)Py_SIZE(self)) lim = Py_SIZE(self) - cur - 1; - memmove(self->ob_item + (cur - i) * itemsize, - self->ob_item + (cur + 1) * itemsize, + memmove(self->data->items + (cur - i) * itemsize, + self->data->items + (cur + 1) * itemsize, lim * itemsize); } cur = start + (size_t)slicelength * step; if (cur < (size_t)Py_SIZE(self)) { - memmove(self->ob_item + (cur-slicelength) * itemsize, - self->ob_item + cur * itemsize, + memmove(self->data->items + (cur-slicelength) * itemsize, + self->data->items + cur * itemsize, (Py_SIZE(self) - cur) * itemsize); } if (array_resize(self, Py_SIZE(self) - slicelength) < 0) @@ -3012,8 +3018,8 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) } for (cur = start, i = 0; i < slicelength; cur += step, i++) { - memcpy(self->ob_item + cur * itemsize, - other->ob_item + i * itemsize, + memcpy(self->data->items + cur * itemsize, + other->data->items + i * itemsize, itemsize); } return 0; @@ -3063,7 +3069,7 @@ array_buffer_getbuf_lock_held(PyObject *op, Py_buffer *view, int flags) return -1; } - view->buf = (void *)self->ob_item; + view->buf = (void *)arraydata_safeitems(self->data); view->obj = Py_NewRef(self); if (view->buf == NULL) view->buf = (void *)emptybuf; @@ -3172,7 +3178,7 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) Py_DECREF(a); return NULL; } - if (setarrayitem(a, i, v, ap->ob_item) != 0) { + if (setarrayitem(ap, i, v, ap->data->items) != 0) { Py_DECREF(v); Py_DECREF(a); return NULL; @@ -3201,22 +3207,15 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) if (n > 0) { arrayobject *self = (arrayobject *)a; -#ifdef Py_GIL_DISABLED - self->ob_item = arrayarray_set(self->ob_item, + self->data = arraydata_set_items(self->data, (char *)ustr, n, sizeof(wchar_t), false); PyMem_Free(ustr); - if (self->ob_item == NULL) { + if (self->data == NULL) { Py_DECREF(a); PyErr_NoMemory(); return NULL; } -#else - // self->ob_item may be NULL but it is safe. - PyMem_Free(self->ob_item); - self->ob_item = (char *)ustr; -#endif Py_SET_SIZE(self, n); - self->allocated = n; } } else { // c == 'w' @@ -3228,28 +3227,21 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) } arrayobject *self = (arrayobject *)a; -#ifdef Py_GIL_DISABLED - self->ob_item = arrayarray_set(self->ob_item, + self->data = arraydata_set_items(self->data, (char *)ustr, n, sizeof(Py_UCS4), false); PyMem_Free(ustr); - if (self->ob_item == NULL) { + if (self->data == NULL) { Py_DECREF(a); PyErr_NoMemory(); return NULL; } -#else - // self->ob_item may be NULL but it is safe. - PyMem_Free(self->ob_item); - self->ob_item = (char *)ustr; -#endif Py_SET_SIZE(self, n); - self->allocated = n; } } else if (initial != NULL && array_Check(initial, state) && len > 0) { arrayobject *self = (arrayobject *)a; arrayobject *other = (arrayobject *)initial; - memcpy(self->ob_item, other->ob_item, len * other->ob_descr->itemsize); + memcpy(self->data->items, other->data->items, len * other->ob_descr->itemsize); } if (it != NULL) { if (array_iter_extend((arrayobject *)a, it) == -1) { @@ -3608,7 +3600,7 @@ static PyType_Spec arrayiter_spec = { /*********************** Install Module **************************/ static int -array_traverse(PyObject *module, visitproc visit, void *arg) +arraymodule_traverse(PyObject *module, visitproc visit, void *arg) { array_state *state = get_array_state(module); Py_VISIT(state->ArrayType); @@ -3618,7 +3610,7 @@ array_traverse(PyObject *module, visitproc visit, void *arg) } static int -array_clear(PyObject *module) +arraymodule_clear(PyObject *module) { array_state *state = get_array_state(module); Py_CLEAR(state->ArrayType); @@ -3632,9 +3624,9 @@ array_clear(PyObject *module) } static void -array_free(void *module) +arraymodule_free(void *module) { - (void)array_clear((PyObject *)module); + (void)arraymodule_clear((PyObject *)module); } /* No functions in array module. */ @@ -3661,7 +3653,7 @@ do { \ } while (0) static int -array_modexec(PyObject *m) +arraymodule_modexec(PyObject *m) { array_state *state = get_array_state(m); char buffer[Py_ARRAY_LENGTH(descriptors)], *p; @@ -3716,7 +3708,7 @@ array_modexec(PyObject *m) } static PyModuleDef_Slot arrayslots[] = { - {Py_mod_exec, array_modexec}, + {Py_mod_exec, arraymodule_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} @@ -3730,9 +3722,9 @@ static struct PyModuleDef arraymodule = { .m_doc = module_doc, .m_methods = a_methods, .m_slots = arrayslots, - .m_traverse = array_traverse, - .m_clear = array_clear, - .m_free = array_free, + .m_traverse = arraymodule_traverse, + .m_clear = arraymodule_clear, + .m_free = arraymodule_free, }; From fb6212a500a785ff95a60d2966c75fba592f710a Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Wed, 5 Mar 2025 13:57:48 -0500 Subject: [PATCH 10/32] test and tsan stuff --- Lib/test/test_array.py | 94 ++++++++++++++++++++++++------------------ Modules/arraymodule.c | 11 ++--- 2 files changed, 60 insertions(+), 45 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 41b52ea424faff..a438ba0a9eb362 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -4,6 +4,7 @@ import collections.abc import io +import os import unittest from test import support from test.support import import_helper @@ -104,7 +105,10 @@ def test_empty(self): class ArrayReconstructorTest(unittest.TestCase): def setUp(self): - self.enterContext(warnings.catch_warnings()) + if (not support.Py_GIL_DISABLED or + any('"parallel_threads": null' in a for a in sys.argv) or + all('parallel_threads' not in a for a in sys.argv)): + self.enterContext(warnings.catch_warnings()) warnings.filterwarnings( "ignore", message="The 'u' type code is deprecated and " @@ -220,7 +224,10 @@ class BaseTest: # minitemsize: the minimum guaranteed itemsize def setUp(self): - self.enterContext(warnings.catch_warnings()) + if (not support.Py_GIL_DISABLED or + any('"parallel_threads": null' in a for a in sys.argv) or + all('parallel_threads' not in a for a in sys.argv)): + self.enterContext(warnings.catch_warnings()) warnings.filterwarnings( "ignore", message="The 'u' type code is deprecated and " @@ -472,54 +479,59 @@ def test_insert(self): def test_tofromfile(self): a = array.array(self.typecode, 2*self.example) self.assertRaises(TypeError, a.tofile) - os_helper.unlink(os_helper.TESTFN) - f = open(os_helper.TESTFN, 'wb') - try: - a.tofile(f) - f.close() - b = array.array(self.typecode) - f = open(os_helper.TESTFN, 'rb') - self.assertRaises(TypeError, b.fromfile) - b.fromfile(f, len(self.example)) - self.assertEqual(b, array.array(self.typecode, self.example)) - self.assertNotEqual(a, b) - self.assertRaises(EOFError, b.fromfile, f, len(self.example)+1) - self.assertEqual(a, b) - f.close() - finally: - if not f.closed: + with os_helper.temp_dir() as temp_dir: + temp_path = os.path.join(temp_dir, os_helper.TESTFN) + f = open(temp_path, 'wb') + try: + a.tofile(f) f.close() - os_helper.unlink(os_helper.TESTFN) + b = array.array(self.typecode) + f = open(temp_path, 'rb') + self.assertRaises(TypeError, b.fromfile) + b.fromfile(f, len(self.example)) + self.assertEqual(b, array.array(self.typecode, self.example)) + self.assertNotEqual(a, b) + self.assertRaises(EOFError, b.fromfile, f, len(self.example)+1) + self.assertEqual(a, b) + f.close() + finally: + if not f.closed: + f.close() + os_helper.unlink(temp_path) def test_fromfile_ioerror(self): # Issue #5395: Check if fromfile raises a proper OSError # instead of EOFError. a = array.array(self.typecode) - f = open(os_helper.TESTFN, 'wb') - try: - self.assertRaises(OSError, a.fromfile, f, len(self.example)) - finally: - f.close() - os_helper.unlink(os_helper.TESTFN) + with os_helper.temp_dir() as temp_dir: + temp_path = os.path.join(temp_dir, os_helper.TESTFN) + f = open(temp_path, 'wb') + try: + self.assertRaises(OSError, a.fromfile, f, len(self.example)) + finally: + f.close() + os_helper.unlink(temp_path) def test_filewrite(self): a = array.array(self.typecode, 2*self.example) - f = open(os_helper.TESTFN, 'wb') - try: - f.write(a) - f.close() - b = array.array(self.typecode) - f = open(os_helper.TESTFN, 'rb') - b.fromfile(f, len(self.example)) - self.assertEqual(b, array.array(self.typecode, self.example)) - self.assertNotEqual(a, b) - b.fromfile(f, len(self.example)) - self.assertEqual(a, b) - f.close() - finally: - if not f.closed: + with os_helper.temp_dir() as temp_dir: + temp_path = os.path.join(temp_dir, os_helper.TESTFN) + f = open(temp_path, 'wb') + try: + f.write(a) + f.close() + b = array.array(self.typecode) + f = open(temp_path, 'rb') + b.fromfile(f, len(self.example)) + self.assertEqual(b, array.array(self.typecode, self.example)) + self.assertNotEqual(a, b) + b.fromfile(f, len(self.example)) + self.assertEqual(a, b) f.close() - os_helper.unlink(os_helper.TESTFN) + finally: + if not f.closed: + f.close() + os_helper.unlink(temp_path) def test_tofromlist(self): a = array.array(self.typecode, 2*self.example) @@ -1201,6 +1213,7 @@ def test_obsolete_write_lock(self): a = array.array('B', b"") self.assertRaises(BufferError, _testcapi.getbuffer_with_null_view, a) + @unittest.skipIf(support.Py_GIL_DISABLED, 'not freed if GIL disabled') def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, array.array, (self.typecode,)) @@ -1255,6 +1268,7 @@ def test_issue17223(self): self.assertRaises(ValueError, a.tounicode) self.assertRaises(ValueError, str, a) + @unittest.skipIf(support.Py_GIL_DISABLED, 'warning stuff is not free-thread safe yet') def test_typecode_u_deprecation(self): with self.assertWarns(DeprecationWarning): array.array("u") diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index d24bba34a0312c..37e0d8f8a3787b 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -254,8 +254,8 @@ array_resize(arrayobject *self, Py_ssize_t newsize) #else arraydata_free(self->data, false); #endif - self->data = NULL; Py_SET_SIZE(self, 0); + FT_ATOMIC_STORE_PTR_RELAXED(self->data, NULL); return 0; } @@ -2629,10 +2629,11 @@ array_array___reduce_ex___impl(arrayobject *self, PyTypeObject *cls, array_state *state = get_array_state_by_class(cls); assert(state != NULL); - if (state->array_reconstructor == NULL) { - state->array_reconstructor = PyImport_ImportModuleAttrString( - "array", "_array_reconstructor"); - if (state->array_reconstructor == NULL) { + if (FT_ATOMIC_LOAD_PTR_RELAXED(state->array_reconstructor) == NULL) { + PyObject *array_reconstructor = PyImport_ImportModuleAttrString( + "array", "_array_reconstructor"); + FT_ATOMIC_STORE_PTR_RELAXED(state->array_reconstructor, array_reconstructor); + if (array_reconstructor == NULL) { return NULL; } } From 04a8f9dc9fe07f432ef206c90e660a0dfe74ac48 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Wed, 5 Mar 2025 18:51:44 -0500 Subject: [PATCH 11/32] getters and setters using atomics --- Lib/test/test_array.py | 161 ++++++++++++++++++++++++----------------- Modules/arraymodule.c | 142 ++++++++++++++++++++++-------------- 2 files changed, 184 insertions(+), 119 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index a438ba0a9eb362..8b759c5929eff8 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -105,10 +105,7 @@ def test_empty(self): class ArrayReconstructorTest(unittest.TestCase): def setUp(self): - if (not support.Py_GIL_DISABLED or - any('"parallel_threads": null' in a for a in sys.argv) or - all('parallel_threads' not in a for a in sys.argv)): - self.enterContext(warnings.catch_warnings()) + self.enterContext(warnings.catch_warnings()) warnings.filterwarnings( "ignore", message="The 'u' type code is deprecated and " @@ -224,10 +221,7 @@ class BaseTest: # minitemsize: the minimum guaranteed itemsize def setUp(self): - if (not support.Py_GIL_DISABLED or - any('"parallel_threads": null' in a for a in sys.argv) or - all('parallel_threads' not in a for a in sys.argv)): - self.enterContext(warnings.catch_warnings()) + self.enterContext(warnings.catch_warnings()) warnings.filterwarnings( "ignore", message="The 'u' type code is deprecated and " @@ -1696,6 +1690,29 @@ class FreeThreadingTest(unittest.TestCase): # Non-deterministic, but at least one of these things will fail if # array module is not free-thread safe. + def setUp(self): + self.enterContext(warnings.catch_warnings()) + warnings.filterwarnings( + "ignore", + message="The 'u' type code is deprecated and " + "will be removed in Python 3.16", + category=DeprecationWarning) + + def check(self, funcs, a=None, *args): + if a is None: + a = array.array('i', [1]) + + barrier = threading.Barrier(len(funcs)) + threads = [] + + for func in funcs: + thread = threading.Thread(target=func, args=(barrier, a, *args)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + @unittest.skipUnless(support.Py_GIL_DISABLED, 'this test can only possibly fail with GIL disabled') @threading_helper.reap_threads @threading_helper.requires_working_threading() @@ -1889,67 +1906,81 @@ def iter_reduce(b, a, it): c = it.__reduce__() assert not c[1] or 0xdd not in c[1][0] - def check(funcs, a=None, *args): - if a is None: - a = array.array('i', [1]) - - barrier = threading.Barrier(len(funcs)) - threads = [] - - for func in funcs: - thread = threading.Thread(target=func, args=(barrier, a, *args)) - - threads.append(thread) - - with threading_helper.start_threads(threads): - pass - - check([pop1] * 10) - check([pop1] + [subscr0] * 10) - check([append1] * 10) - check([insert1] * 10) - check([pop1] + [index1] * 10) - check([pop1] + [contains1] * 10) - check([insert1] + [repeat2] * 10) - check([pop1] + [repr1] * 10) - check([inplace_repeat2] * 10) - check([byteswap] * 10) - check([insert1] + [clear] * 10) - check([pop1] + [count1] * 10) - check([remove1] * 10) - check([clear] + [copy] * 10, array.array('B', b'0' * 0x400000)) - check([pop1] + [reduce_ex2] * 10) - check([clear] + [reduce_ex3] * 10, array.array('B', b'0' * 0x400000)) - check([pop1] + [tobytes] * 10) - check([pop1] + [tolist] * 10) - check([clear, tounicode] * 10, array.array('w', 'a'*10000)) - check([clear, tofile] * 10, array.array('w', 'a'*10000)) - check([clear] + [extend] * 10) - check([clear] + [inplace_concat] * 10) - check([clear] + [concat] * 10, array.array('w', 'a'*10000)) - check([fromunicode] * 10, array.array('w', 'a')) - check([frombytes] * 10) - check([fromlist] * 10) - check([clear] + [richcmplhs] * 10, array.array('i', [1]*10000)) - check([clear] + [richcmprhs] * 10, array.array('i', [1]*10000)) - check([clear, ass0] * 10, array.array('i', [1]*10000)) # to test array_ass_item must disable Py_mp_ass_subscript - check([clear] + [new] * 10, array.array('w', 'a'*10000)) - check([clear] + [repr_] * 10, array.array('B', b'0' * 0x40000)) - check([clear] + [repr_] * 10, array.array('B', b'0' * 0x40000)) - check([clear] + [irepeat] * 10, array.array('B', b'0' * 0x40000)) - check([clear] + [iter_reduce] * 10, a := array.array('B', b'0' * 0x400), iter(a)) + self.check([pop1] * 10) + self.check([pop1] + [subscr0] * 10) + self.check([append1] * 10) + self.check([insert1] * 10) + self.check([pop1] + [index1] * 10) + self.check([pop1] + [contains1] * 10) + self.check([insert1] + [repeat2] * 10) + self.check([pop1] + [repr1] * 10) + self.check([inplace_repeat2] * 10) + self.check([byteswap] * 10) + self.check([insert1] + [clear] * 10) + self.check([pop1] + [count1] * 10) + self.check([remove1] * 10) + self.check([clear] + [copy] * 10, array.array('B', b'0' * 0x400000)) + self.check([pop1] + [reduce_ex2] * 10) + self.check([clear] + [reduce_ex3] * 10, array.array('B', b'0' * 0x400000)) + self.check([pop1] + [tobytes] * 10) + self.check([pop1] + [tolist] * 10) + self.check([clear, tounicode] * 10, array.array('w', 'a'*10000)) + self.check([clear, tofile] * 10, array.array('w', 'a'*10000)) + self.check([clear] + [extend] * 10) + self.check([clear] + [inplace_concat] * 10) + self.check([clear] + [concat] * 10, array.array('w', 'a'*10000)) + self.check([fromunicode] * 10, array.array('w', 'a')) + self.check([frombytes] * 10) + self.check([fromlist] * 10) + self.check([clear] + [richcmplhs] * 10, array.array('i', [1]*10000)) + self.check([clear] + [richcmprhs] * 10, array.array('i', [1]*10000)) + self.check([clear, ass0] * 10, array.array('i', [1]*10000)) # to test array_ass_item must disable Py_mp_ass_subscript + self.check([clear] + [new] * 10, array.array('w', 'a'*10000)) + self.check([clear] + [repr_] * 10, array.array('B', b'0' * 0x40000)) + self.check([clear] + [repr_] * 10, array.array('B', b'0' * 0x40000)) + self.check([clear] + [irepeat] * 10, array.array('B', b'0' * 0x40000)) + self.check([clear] + [iter_reduce] * 10, a := array.array('B', b'0' * 0x400), iter(a)) # make sure we handle non-self objects correctly - check([clear] + [newi] * 10, [2] * random.randint(0, 100)) - check([fromlistlclear] + [fromlistl] * 10, array.array('i', [1]), [2] * random.randint(0, 100)) - check([clear2] + [concat2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) - check([clear2] + [inplace_concat2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) - check([clear2] + [extend2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) - check([clear2] + [ass_subscr2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) - check([clear2] + [frombytes2] * 10, array.array('w', 'a'*10000), array.array('B', b'a'*10000)) + self.check([clear] + [newi] * 10, [2] * random.randint(0, 100)) + self.check([fromlistlclear] + [fromlistl] * 10, array.array('i', [1]), [2] * random.randint(0, 100)) + self.check([clear2] + [concat2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + self.check([clear2] + [inplace_concat2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + self.check([clear2] + [extend2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + self.check([clear2] + [ass_subscr2] * 10, array.array('w', 'a'*10000), array.array('w', 'a'*10000)) + self.check([clear2] + [frombytes2] * 10, array.array('w', 'a'*10000), array.array('B', b'a'*10000)) # iterator stuff - check([clear] + [iter_next] * 10, a := array.array('i', [1] * 10), iter(a)) + self.check([clear] + [iter_next] * 10, a := array.array('i', [1] * 10), iter(a)) + + @unittest.skipUnless(support.check_sanitizer(thread=True), 'meant for tsan') + def test_free_threading_tsan(self): + def copy_back_and_forth(b, a, count): + b.wait() + for _ in range(count): + a[0] = a[1] + a[1] = a[0] + + def extend_range(b, a, count): + b.wait() + for _ in range(count): + a.extend(range(10)) + + def append_and_pop(b, a, count): + b.wait() + for _ in range(count): + a.append(1) + a.pop() + + for tc in typecodes: + if tc in 'uw': + a = array.array(tc, 'ab') + else: + a = array.array(tc, [0, 1]) + + self.check([copy_back_and_forth] * 10, a, 100) + self.check([append_and_pop] * 10, a, 100) + self.check([copy_back_and_forth] * 10 + [extend_range], a, 10) if __name__ == "__main__": diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 37e0d8f8a3787b..3cd7c645032ce7 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -321,8 +321,8 @@ in bounds; that's the responsibility of the caller. static PyObject * b_getitem(char *items, Py_ssize_t i) { - long x = ((signed char *)items)[i]; - return PyLong_FromLong(x); + return PyLong_FromLong( + (long) (signed char) FT_ATOMIC_LOAD_CHAR_RELAXED(items[i])); } static int @@ -332,8 +332,9 @@ b_setitem(char *items, Py_ssize_t i, PyObject *v) /* PyArg_Parse's 'b' formatter is for an unsigned char, therefore must use the next size up that is signed ('h') and manually do the overflow checking */ - if (!PyArg_Parse(v, "h;array item must be integer", &x)) + if (!PyArg_Parse(v, "h;array item must be integer", &x)) { return -1; + } else if (x < -128) { PyErr_SetString(PyExc_OverflowError, "signed char is less than minimum"); @@ -344,16 +345,17 @@ b_setitem(char *items, Py_ssize_t i, PyObject *v) "signed char is greater than maximum"); return -1; } - if (i >= 0) - ((char *)items)[i] = (char)x; + if (i >= 0) { + FT_ATOMIC_STORE_CHAR_RELAXED(items[i], (char) x); + } return 0; } static PyObject * BB_getitem(char *items, Py_ssize_t i) { - long x = ((unsigned char *)items)[i]; - return PyLong_FromLong(x); + return PyLong_FromLong( + (long) (unsigned char) FT_ATOMIC_LOAD_CHAR_RELAXED(items[i])); } static int @@ -361,17 +363,20 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) { unsigned char x; /* 'B' == unsigned char, maps to PyArg_Parse's 'b' formatter */ - if (!PyArg_Parse(v, "b;array item must be integer", &x)) + if (!PyArg_Parse(v, "b;array item must be integer", &x)) { return -1; - if (i >= 0) - ((unsigned char *)items)[i] = x; + } + if (i >= 0) { + FT_ATOMIC_STORE_CHAR_RELAXED(items[i], x); + } return 0; } static PyObject * u_getitem(char *items, Py_ssize_t i) { - return PyUnicode_FromOrdinal(((wchar_t *) items)[i]); + return PyUnicode_FromOrdinal( + (wchar_t) FT_ATOMIC_LOAD_INT_RELAXED(((wchar_t *) items)[i])); } static int @@ -406,7 +411,7 @@ u_setitem(char *items, Py_ssize_t i, PyObject *v) assert(len == 1); if (i >= 0) { - ((wchar_t *)items)[i] = w; + FT_ATOMIC_STORE_INT_RELAXED(((wchar_t *) items)[i], w); } return 0; } @@ -414,7 +419,8 @@ u_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * w_getitem(char *items, Py_ssize_t i) { - return PyUnicode_FromOrdinal(((Py_UCS4 *) items)[i]); + return PyUnicode_FromOrdinal( + (Py_UCS4) FT_ATOMIC_LOAD_UINT32_RELAXED(((Py_UCS4 *) items)[i])); } static int @@ -436,7 +442,8 @@ w_setitem(char *items, Py_ssize_t i, PyObject *v) } if (i >= 0) { - ((Py_UCS4 *)items)[i] = PyUnicode_READ_CHAR(v, 0); + Py_UCS4 w = PyUnicode_READ_CHAR(v, 0); + FT_ATOMIC_STORE_UINT32_RELAXED(((Py_UCS4 *) items)[i], w); } return 0; } @@ -444,7 +451,8 @@ w_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * h_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong((long) ((short *)items)[i]); + return PyLong_FromLong( + (long) FT_ATOMIC_LOAD_SHORT_RELAXED(((short *) items)[i])); } @@ -453,17 +461,20 @@ h_setitem(char *items, Py_ssize_t i, PyObject *v) { short x; /* 'h' == signed short, maps to PyArg_Parse's 'h' formatter */ - if (!PyArg_Parse(v, "h;array item must be integer", &x)) + if (!PyArg_Parse(v, "h;array item must be integer", &x)) { return -1; - if (i >= 0) - ((short *)items)[i] = x; + } + if (i >= 0) { + FT_ATOMIC_STORE_SHORT_RELAXED(((short *) items)[i], x); + } return 0; } static PyObject * HH_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong((long) ((unsigned short *)items)[i]); + return PyLong_FromLong( + (long) (unsigned short) FT_ATOMIC_LOAD_SHORT_RELAXED(((short *) items)[i])); } static int @@ -472,8 +483,9 @@ HH_setitem(char *items, Py_ssize_t i, PyObject *v) int x; /* PyArg_Parse's 'h' formatter is for a signed short, therefore must use the next size up and manually do the overflow checking */ - if (!PyArg_Parse(v, "i;array item must be integer", &x)) + if (!PyArg_Parse(v, "i;array item must be integer", &x)) { return -1; + } else if (x < 0) { PyErr_SetString(PyExc_OverflowError, "unsigned short is less than minimum"); @@ -484,15 +496,17 @@ HH_setitem(char *items, Py_ssize_t i, PyObject *v) "unsigned short is greater than maximum"); return -1; } - if (i >= 0) - ((short *)items)[i] = (short)x; + if (i >= 0) { + FT_ATOMIC_STORE_SHORT_RELAXED(((short *) items)[i], (unsigned short) x); + } return 0; } static PyObject * i_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong((long) ((int *)items)[i]); + return PyLong_FromLong( + (long) FT_ATOMIC_LOAD_INT_RELAXED(((int *) items)[i])); } static int @@ -500,10 +514,12 @@ i_setitem(char *items, Py_ssize_t i, PyObject *v) { int x; /* 'i' == signed int, maps to PyArg_Parse's 'i' formatter */ - if (!PyArg_Parse(v, "i;array item must be integer", &x)) + if (!PyArg_Parse(v, "i;array item must be integer", &x)) { return -1; - if (i >= 0) - ((int *)items)[i] = x; + } + if (i >= 0) { + FT_ATOMIC_STORE_INT_RELAXED(((int *)items)[i], x); + } return 0; } @@ -511,7 +527,7 @@ static PyObject * II_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLong( - (unsigned long) ((unsigned int *)items)[i]); + (unsigned long) (unsigned int) FT_ATOMIC_LOAD_INT_RELAXED(((int *) items)[i])); } static int @@ -542,8 +558,9 @@ II_setitem(char *items, Py_ssize_t i, PyObject *v) } return -1; } - if (i >= 0) - ((unsigned int *)items)[i] = (unsigned int)x; + if (i >= 0) { + FT_ATOMIC_STORE_INT_RELAXED(((int *) items)[i], (unsigned int) x); + } if (do_decref) { Py_DECREF(v); @@ -554,24 +571,28 @@ II_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * l_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong(((long *)items)[i]); + return PyLong_FromLong( + FT_ATOMIC_LOAD_LONG_RELAXED(((long *) items)[i])); } static int l_setitem(char *items, Py_ssize_t i, PyObject *v) { long x; - if (!PyArg_Parse(v, "l;array item must be integer", &x)) + if (!PyArg_Parse(v, "l;array item must be integer", &x)) { return -1; - if (i >= 0) - ((long *)items)[i] = x; + } + if (i >= 0) { + FT_ATOMIC_STORE_LONG_RELAXED(((long *) items)[i], x); + } return 0; } static PyObject * LL_getitem(char *items, Py_ssize_t i) { - return PyLong_FromUnsignedLong(((unsigned long *)items)[i]); + return PyLong_FromUnsignedLong( + (unsigned long) FT_ATOMIC_LOAD_LONG_RELAXED(((long *) items)[i])); } static int @@ -594,8 +615,9 @@ LL_setitem(char *items, Py_ssize_t i, PyObject *v) } return -1; } - if (i >= 0) - ((unsigned long *)items)[i] = x; + if (i >= 0) { + FT_ATOMIC_STORE_LONG_RELAXED(((long *) items)[i], x); + } if (do_decref) { Py_DECREF(v); @@ -606,17 +628,20 @@ LL_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * q_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLongLong(((long long *)items)[i]); + return PyLong_FromLongLong( + FT_ATOMIC_LOAD_LLONG_RELAXED(((long long *) items)[i])); } static int q_setitem(char *items, Py_ssize_t i, PyObject *v) { long long x; - if (!PyArg_Parse(v, "L;array item must be integer", &x)) + if (!PyArg_Parse(v, "L;array item must be integer", &x)) { return -1; - if (i >= 0) - ((long long *)items)[i] = x; + } + if (i >= 0) { + FT_ATOMIC_STORE_LLONG_RELAXED(((long long *) items)[i], x); + } return 0; } @@ -624,7 +649,7 @@ static PyObject * QQ_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLongLong( - ((unsigned long long *)items)[i]); + (unsigned long long) FT_ATOMIC_LOAD_LLONG_RELAXED(((long long *) items)[i])); } static int @@ -647,8 +672,9 @@ QQ_setitem(char *items, Py_ssize_t i, PyObject *v) } return -1; } - if (i >= 0) - ((unsigned long long *)items)[i] = x; + if (i >= 0) { + FT_ATOMIC_STORE_LLONG_RELAXED(((long long *) items)[i], x); + } if (do_decref) { Py_DECREF(v); @@ -659,34 +685,40 @@ QQ_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * f_getitem(char *items, Py_ssize_t i) { - return PyFloat_FromDouble((double) ((float *)items)[i]); + return PyFloat_FromDouble( + (double) FT_ATOMIC_LOAD_FLOAT_RELAXED(((float *) items)[i])); } static int f_setitem(char *items, Py_ssize_t i, PyObject *v) { float x; - if (!PyArg_Parse(v, "f;array item must be float", &x)) + if (!PyArg_Parse(v, "f;array item must be float", &x)) { return -1; - if (i >= 0) - ((float *)items)[i] = x; + } + if (i >= 0) { + FT_ATOMIC_STORE_FLOAT_RELAXED(((float *) items)[i], x); + } return 0; } static PyObject * d_getitem(char *items, Py_ssize_t i) { - return PyFloat_FromDouble(((double *)items)[i]); + return PyFloat_FromDouble( + FT_ATOMIC_LOAD_DOUBLE_RELAXED(((double *) items)[i])); } static int d_setitem(char *items, Py_ssize_t i, PyObject *v) { double x; - if (!PyArg_Parse(v, "d;array item must be float", &x)) + if (!PyArg_Parse(v, "d;array item must be float", &x)) { return -1; - if (i >= 0) - ((double *)items)[i] = x; + } + if (i >= 0) { + FT_ATOMIC_STORE_DOUBLE_RELAXED(((double *) items)[i], x); + } return 0; } @@ -695,9 +727,11 @@ d_setitem(char *items, Py_ssize_t i, PyObject *v) code##_compareitems(const void *lhs, const void *rhs, Py_ssize_t length) \ { \ const type *a = lhs, *b = rhs; \ - for (Py_ssize_t i = 0; i < length; ++i) \ - if (a[i] != b[i]) \ + for (Py_ssize_t i = 0; i < length; ++i) { \ + if (a[i] != b[i]) { \ return a[i] < b[i] ? -1 : 1; \ + } \ + } \ return 0; \ } @@ -910,7 +944,7 @@ setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) goto error; } arrayobject *ap = (arrayobject *)op; - arraydata *data = _Py_atomic_load_ptr_relaxed(&ap->data); + arraydata *data = _Py_atomic_load_ptr_acquire(&ap->data); if (data == NULL) { goto error; } From 01423fd9f58b4be5fce171ce580443444b0c5799 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 6 Mar 2025 08:26:59 -0500 Subject: [PATCH 12/32] fix 2 byte wchar_t --- Modules/arraymodule.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 3cd7c645032ce7..bef93e7cdf6b7a 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -375,8 +375,13 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * u_getitem(char *items, Py_ssize_t i) { +#if WCHAR_MAX == 0x7FFFFFFF return PyUnicode_FromOrdinal( (wchar_t) FT_ATOMIC_LOAD_INT_RELAXED(((wchar_t *) items)[i])); +#else + return PyUnicode_FromOrdinal( + (wchar_t) FT_ATOMIC_LOAD_SHORT_RELAXED(((wchar_t *) items)[i])); +#endif } static int @@ -411,7 +416,11 @@ u_setitem(char *items, Py_ssize_t i, PyObject *v) assert(len == 1); if (i >= 0) { +#if WCHAR_MAX == 0x7FFFFFFF FT_ATOMIC_STORE_INT_RELAXED(((wchar_t *) items)[i], w); +#else + FT_ATOMIC_STORE_SHORT_RELAXED(((wchar_t *) items)[i], w); +#endif } return 0; } @@ -3235,6 +3244,7 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) if (c == 'u') { Py_ssize_t n; wchar_t *ustr = PyUnicode_AsWideCharString(initial, &n); + printf("... %ld, %ld\n", sizeof(wchar_t), n); if (ustr == NULL) { Py_DECREF(a); return NULL; From 99331dd257b892e842df4af5c61363f664ce6e30 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 6 Mar 2025 08:42:27 -0500 Subject: [PATCH 13/32] remove debug printf --- Modules/arraymodule.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index bef93e7cdf6b7a..72eaf37bfb4a71 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -3244,7 +3244,6 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) if (c == 'u') { Py_ssize_t n; wchar_t *ustr = PyUnicode_AsWideCharString(initial, &n); - printf("... %ld, %ld\n", sizeof(wchar_t), n); if (ustr == NULL) { Py_DECREF(a); return NULL; From 07c98cc6d5e9d7d3b2218121072b360ba9ae29d0 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 6 Mar 2025 09:13:34 -0500 Subject: [PATCH 14/32] just a hunch... --- Modules/arraymodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 72eaf37bfb4a71..1fdb720b8cc9e1 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -375,7 +375,7 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * u_getitem(char *items, Py_ssize_t i) { -#if WCHAR_MAX == 0x7FFFFFFF +#if WCHAR_MAX > 0xFFFF return PyUnicode_FromOrdinal( (wchar_t) FT_ATOMIC_LOAD_INT_RELAXED(((wchar_t *) items)[i])); #else @@ -416,7 +416,7 @@ u_setitem(char *items, Py_ssize_t i, PyObject *v) assert(len == 1); if (i >= 0) { -#if WCHAR_MAX == 0x7FFFFFFF +#if WCHAR_MAX > 0xFFFF FT_ATOMIC_STORE_INT_RELAXED(((wchar_t *) items)[i], w); #else FT_ATOMIC_STORE_SHORT_RELAXED(((wchar_t *) items)[i], w); From cf3bbbb508205de823500867949e4cdf1c4a64a6 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 6 Mar 2025 17:39:18 -0500 Subject: [PATCH 15/32] atomic "memcpy" --- Lib/test/libregrtest/tsan.py | 1 + Lib/test/test_array.py | 30 +++++++++++++++++----- Modules/arraymodule.c | 50 ++++++++++++++++++++++++++++++++---- 3 files changed, 70 insertions(+), 11 deletions(-) diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index 77dde5ecb076e2..1fd92497ab954e 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -2,6 +2,7 @@ # chosen because they use threads and run in a reasonable amount of time. TSAN_TESTS = [ + 'test_array', # TODO: enable more of test_capi once bugs are fixed (GH-116908, GH-116909). 'test_capi.test_mem', 'test_capi.test_pyatomic', diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 8b759c5929eff8..02feab8e119953 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1961,16 +1961,30 @@ def copy_back_and_forth(b, a, count): a[0] = a[1] a[1] = a[0] + def append_and_pop(b, a, count): + v = a[0] + b.wait() + for _ in range(count): + a.append(v) + a.pop() + + def copy_random(b, a, count): + end = len(a) - 1 + idxs = [(random.randint(0, end), random.randint(0, end)) for _ in range(count)] + b.wait() + for src, dst in idxs: + a[dst] = a[src] + def extend_range(b, a, count): b.wait() for _ in range(count): a.extend(range(10)) - def append_and_pop(b, a, count): + def extend_self(b, a, count): + c = a[:] b.wait() for _ in range(count): - a.append(1) - a.pop() + a.extend(c) for tc in typecodes: if tc in 'uw': @@ -1978,9 +1992,13 @@ def append_and_pop(b, a, count): else: a = array.array(tc, [0, 1]) - self.check([copy_back_and_forth] * 10, a, 100) - self.check([append_and_pop] * 10, a, 100) - self.check([copy_back_and_forth] * 10 + [extend_range], a, 10) + self.check([copy_back_and_forth] * 10, a, 100) + self.check([append_and_pop] * 10, a, 100) + self.check([copy_back_and_forth] * 10 + [extend_self], a, 100) + + if tc not in 'uw': + self.check([copy_back_and_forth] * 10 + [extend_range], a, 100) + self.check([copy_random] * 10, a * 5, 100) if __name__ == "__main__": diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 1fdb720b8cc9e1..3b4ecf128c4340 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -197,7 +197,44 @@ arraydata_set_items(arraydata *data, char *newitems, Py_ssize_t newsize, int ite return data; } +#ifdef Py_GIL_DISABLED + +static void +atomic_itemcpy(void *dest, const void *src, size_t n, int itemsize) +{ + if (itemsize == 1) { + for (char *d = (char *) dest, *end = d + n, *s = (char *) src; + d < end; d++, s++) { + _Py_atomic_store_char_relaxed(d, _Py_atomic_load_char_relaxed(s)); + } + } + else if (itemsize == 2) { + for (short *d = (short *) dest, *end = d + n, *s = (short *) src; + d < end; d++, s++) { + _Py_atomic_store_short_relaxed(d, _Py_atomic_load_short_relaxed(s)); + } + } + else if (itemsize == 4) { + for (PY_UINT32_T *d = (PY_UINT32_T *) dest, *end = d + n, *s = (PY_UINT32_T *) src; + d < end; d++, s++) { + _Py_atomic_store_uint32_relaxed(d, _Py_atomic_load_uint32_relaxed(s)); + } + } + else if (itemsize == 8) { + for (PY_UINT64_T *d = (PY_UINT64_T *) dest, *end = d + n, *s = (PY_UINT64_T *) src; + d < end; d++, s++) { + _Py_atomic_store_uint64_relaxed(d, _Py_atomic_load_uint64_relaxed(s)); + } + } + else { + assert(false); + } +} + +#endif + #ifndef Py_GIL_DISABLED + static arraydata * arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) { @@ -211,6 +248,7 @@ arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) data->allocated = size; return data; } + #endif static int @@ -289,7 +327,11 @@ array_resize(arrayobject *self, Py_ssize_t newsize) } if (data != NULL) { Py_ssize_t size = Py_SIZE(self); +#ifdef Py_GIL_DISABLED + atomic_itemcpy(newdata->items, data->items, Py_MIN(size, newsize), itemsize); +#else memcpy(newdata->items, data->items, Py_MIN(size, newsize) * itemsize); +#endif arraydata_free(data, _PyObject_GC_IS_SHARED(self)); } _Py_atomic_store_ptr_release(&self->data, newdata); @@ -375,7 +417,7 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * u_getitem(char *items, Py_ssize_t i) { -#if WCHAR_MAX > 0xFFFF +#if SIZEOF_WCHAR_T > 2 return PyUnicode_FromOrdinal( (wchar_t) FT_ATOMIC_LOAD_INT_RELAXED(((wchar_t *) items)[i])); #else @@ -416,7 +458,7 @@ u_setitem(char *items, Py_ssize_t i, PyObject *v) assert(len == 1); if (i >= 0) { -#if WCHAR_MAX > 0xFFFF +#if SIZEOF_WCHAR_T > 2 FT_ATOMIC_STORE_INT_RELAXED(((wchar_t *) items)[i], w); #else FT_ATOMIC_STORE_SHORT_RELAXED(((wchar_t *) items)[i], w); @@ -960,9 +1002,7 @@ setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) if (!valid_index(i, data->allocated)) { goto error; } - int ret = setarrayitem(ap, i, v, data->items); - _Py_atomic_fence_release(); - return ret; + return setarrayitem(ap, i, v, data->items); error: PyErr_SetString(PyExc_IndexError, "array index out of range"); From 51135a497933e4ac86652e9365a7b6dcda5d51b7 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Fri, 7 Mar 2025 02:15:43 -0500 Subject: [PATCH 16/32] misc --- Modules/arraymodule.c | 44 +++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 3b4ecf128c4340..c83da9b35a48cc 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -197,6 +197,24 @@ arraydata_set_items(arraydata *data, char *newitems, Py_ssize_t newsize, int ite return data; } +#ifndef Py_GIL_DISABLED + +static arraydata * +arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) +{ + if (size > PY_SSIZE_T_MAX/itemsize - 1) { + return NULL; + } + data = (arraydata *)PyMem_Realloc(data, sizeof(arraydata) + size * itemsize); + if (data == NULL) { + return NULL; + } + data->allocated = size; + return data; +} + +#endif + #ifdef Py_GIL_DISABLED static void @@ -205,25 +223,25 @@ atomic_itemcpy(void *dest, const void *src, size_t n, int itemsize) if (itemsize == 1) { for (char *d = (char *) dest, *end = d + n, *s = (char *) src; d < end; d++, s++) { - _Py_atomic_store_char_relaxed(d, _Py_atomic_load_char_relaxed(s)); + *d = _Py_atomic_load_char_relaxed(s); } } else if (itemsize == 2) { for (short *d = (short *) dest, *end = d + n, *s = (short *) src; d < end; d++, s++) { - _Py_atomic_store_short_relaxed(d, _Py_atomic_load_short_relaxed(s)); + *d = _Py_atomic_load_short_relaxed(s); } } else if (itemsize == 4) { for (PY_UINT32_T *d = (PY_UINT32_T *) dest, *end = d + n, *s = (PY_UINT32_T *) src; d < end; d++, s++) { - _Py_atomic_store_uint32_relaxed(d, _Py_atomic_load_uint32_relaxed(s)); + *d = (PY_UINT32_T) _Py_atomic_load_uint32_relaxed(s); } } else if (itemsize == 8) { for (PY_UINT64_T *d = (PY_UINT64_T *) dest, *end = d + n, *s = (PY_UINT64_T *) src; d < end; d++, s++) { - _Py_atomic_store_uint64_relaxed(d, _Py_atomic_load_uint64_relaxed(s)); + *d = (PY_UINT64_T) _Py_atomic_load_uint64_relaxed(s); } } else { @@ -233,24 +251,6 @@ atomic_itemcpy(void *dest, const void *src, size_t n, int itemsize) #endif -#ifndef Py_GIL_DISABLED - -static arraydata * -arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) -{ - if (size > PY_SSIZE_T_MAX/itemsize - 1) { - return NULL; - } - data = (arraydata *)PyMem_Realloc(data, sizeof(arraydata) + size * itemsize); - if (data == NULL) { - return NULL; - } - data->allocated = size; - return data; -} - -#endif - static int array_resize(arrayobject *self, Py_ssize_t newsize) { From fff827e0525e7f9cd3d414ed8cd848e0283795f2 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sat, 8 Mar 2025 17:29:40 -0500 Subject: [PATCH 17/32] use proper type FT_ macros --- Lib/test/test_array.py | 2 ++ Modules/arraymodule.c | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 02feab8e119953..bb2203de627f52 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1954,6 +1954,8 @@ def iter_reduce(b, a, it): self.check([clear] + [iter_next] * 10, a := array.array('i', [1] * 10), iter(a)) @unittest.skipUnless(support.check_sanitizer(thread=True), 'meant for tsan') + @threading_helper.reap_threads + @threading_helper.requires_working_threading() def test_free_threading_tsan(self): def copy_back_and_forth(b, a, count): b.wait() diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index c83da9b35a48cc..2f7444d063b8b0 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -397,7 +397,7 @@ static PyObject * BB_getitem(char *items, Py_ssize_t i) { return PyLong_FromLong( - (long) (unsigned char) FT_ATOMIC_LOAD_CHAR_RELAXED(items[i])); + (long) FT_ATOMIC_LOAD_UCHAR_RELAXED(((unsigned char *) items)[i])); } static int @@ -409,7 +409,7 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_CHAR_RELAXED(items[i], x); + FT_ATOMIC_STORE_UCHAR_RELAXED(((unsigned char *) items)[i], x); } return 0; } @@ -525,7 +525,7 @@ static PyObject * HH_getitem(char *items, Py_ssize_t i) { return PyLong_FromLong( - (long) (unsigned short) FT_ATOMIC_LOAD_SHORT_RELAXED(((short *) items)[i])); + (long) FT_ATOMIC_LOAD_USHORT_RELAXED(((unsigned short *) items)[i])); } static int @@ -548,7 +548,7 @@ HH_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_SHORT_RELAXED(((short *) items)[i], (unsigned short) x); + FT_ATOMIC_STORE_USHORT_RELAXED(((unsigned short *) items)[i], (unsigned short) x); } return 0; } @@ -578,7 +578,7 @@ static PyObject * II_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLong( - (unsigned long) (unsigned int) FT_ATOMIC_LOAD_INT_RELAXED(((int *) items)[i])); + (unsigned long) FT_ATOMIC_LOAD_UINT_RELAXED(((unsigned int *) items)[i])); } static int @@ -610,7 +610,7 @@ II_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_INT_RELAXED(((int *) items)[i], (unsigned int) x); + FT_ATOMIC_STORE_UINT_RELAXED(((unsigned int *) items)[i], (unsigned int) x); } if (do_decref) { @@ -643,7 +643,7 @@ static PyObject * LL_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLong( - (unsigned long) FT_ATOMIC_LOAD_LONG_RELAXED(((long *) items)[i])); + FT_ATOMIC_LOAD_ULONG_RELAXED(((unsigned long *) items)[i])); } static int @@ -667,7 +667,7 @@ LL_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_LONG_RELAXED(((long *) items)[i], x); + FT_ATOMIC_STORE_ULONG_RELAXED(((unsigned long *) items)[i], x); } if (do_decref) { @@ -700,7 +700,7 @@ static PyObject * QQ_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLongLong( - (unsigned long long) FT_ATOMIC_LOAD_LLONG_RELAXED(((long long *) items)[i])); + FT_ATOMIC_LOAD_ULLONG_RELAXED(((unsigned long long *) items)[i])); } static int @@ -724,7 +724,7 @@ QQ_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_LLONG_RELAXED(((long long *) items)[i], x); + FT_ATOMIC_STORE_ULLONG_RELAXED(((unsigned long long *) items)[i], x); } if (do_decref) { From 12f0ff64afe5584982c3518bdf199607595aacd7 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 10 Mar 2025 19:18:31 -0400 Subject: [PATCH 18/32] atomic aggregate _Py_atomic_source_memcpy_relaxed() --- Modules/arraymodule.c | 149 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 122 insertions(+), 27 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 2f7444d063b8b0..d348d313c8c798 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -217,38 +217,136 @@ arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) #ifdef Py_GIL_DISABLED +// This really doesn't belong here, for show at the moment. static void -atomic_itemcpy(void *dest, const void *src, size_t n, int itemsize) +_Py_atomic_source_memcpy_relaxed(void *dest, void *src, size_t n) { - if (itemsize == 1) { - for (char *d = (char *) dest, *end = d + n, *s = (char *) src; - d < end; d++, s++) { - *d = _Py_atomic_load_char_relaxed(s); + int diff = (int)((uintptr_t)dest ^ (uintptr_t)src); + + // the first half is needed to deal with misalignment + + if (diff & 1) { // dest and src not word aligned with each other + for (void *end = (char *)dest + n; dest < end; + dest = (char *)dest + 1, src = (char *)src + 1) { + *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); } + + return; } - else if (itemsize == 2) { - for (short *d = (short *) dest, *end = d + n, *s = (short *) src; - d < end; d++, s++) { - *d = _Py_atomic_load_short_relaxed(s); + + if ((uintptr_t)dest & 1) { // dest and src not word aligned in memory + if (n) { + *(char *)dest = _Py_atomic_load_char_relaxed((char *)src); + dest = (char *)dest + 1; + src = (char *)src + 1; + n -= 1; + } + + if (!n) { + return; } } - else if (itemsize == 4) { - for (PY_UINT32_T *d = (PY_UINT32_T *) dest, *end = d + n, *s = (PY_UINT32_T *) src; - d < end; d++, s++) { - *d = (PY_UINT32_T) _Py_atomic_load_uint32_relaxed(s); + + if (diff & 2) { // dest and src not dword aligned with each other + size_t n2 = n / 2; + + for (void *end = (short *)dest + n2; dest < end; + dest = (short *)dest + 1, src = (short *)src + 1) { + *((short *)dest) = _Py_atomic_load_short_relaxed((short *)src); + } + + if (n & 1) { + *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); } + + return; } - else if (itemsize == 8) { - for (PY_UINT64_T *d = (PY_UINT64_T *) dest, *end = d + n, *s = (PY_UINT64_T *) src; - d < end; d++, s++) { - *d = (PY_UINT64_T) _Py_atomic_load_uint64_relaxed(s); + + if ((uintptr_t)dest & 2) { // dest and src not dword aligned in memory + if (n >= 2) { + *(short *)dest = _Py_atomic_load_short_relaxed((short *)src); + dest = (short *)dest + 1; + src = (short *)src + 1; + n -= 2; + } + + if (!n) { + return; } } - else { - assert(false); + + if (diff & 4) { // dest and src not qword aligned with each other + size_t n4 = n / 4; + + for (void *end = (PY_UINT32_T *)dest + n4; dest < end; + dest = (PY_UINT32_T *)dest + 1, src = (PY_UINT32_T *)src + 1) { + *((PY_UINT32_T *)dest) = (PY_UINT32_T)_Py_atomic_load_uint32_relaxed((PY_UINT32_T *)src); + } + + if (n & 2) { + *((short *)dest) = _Py_atomic_load_short_relaxed((short *)src); + dest = (short *)dest + 1; + src = (short *)src + 1; + } + + if (n & 1) { + *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); + } + + return; + } + + if ((uintptr_t)dest & 4) { // dest and src not qword aligned in memory + if (n >= 4) { + *(PY_UINT32_T *)dest = _Py_atomic_load_uint32_relaxed((PY_UINT32_T *)src); + dest = (PY_UINT32_T *)dest + 1; + src = (PY_UINT32_T *)src + 1; + n -= 4; + } + + if (!n) { + return; + } + } + + // the second half is aligned copy + + size_t n8 = n / 8; + + if (n8) { + for (void *end = (PY_UINT64_T *)dest + n8; dest < end; + dest = (PY_UINT64_T *)dest + 1, src = (PY_UINT64_T *)src + 1) { + *((PY_UINT64_T *)dest) = (PY_UINT64_T)_Py_atomic_load_uint64_relaxed((PY_UINT64_T *)src); + } + + n -= n8 * 8; + } + + if (n & 4) { + *((PY_UINT32_T *)dest) = (PY_UINT32_T)_Py_atomic_load_uint32_relaxed((PY_UINT32_T *)src); + dest = (PY_UINT32_T *)dest + 1; + src = (PY_UINT32_T *)src + 1; + } + + if (n & 2) { + *((short *)dest) = _Py_atomic_load_short_relaxed((short *)src); + dest = (short *)dest + 1; + src = (short *)src + 1; + } + + if (n & 1) { + *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); } } +#define FT_ATOMIC_SOURCE_MEMCPY_RELAXED(dest, src, n) \ + _Py_atomic_source_memcpy_relaxed((dest), (src), (n)) + +#else + +#define FT_ATOMIC_SOURCE_MEMCPY_RELAXED(dest, src, n) \ + memcpy((dest), (src), (n)) + #endif static int @@ -327,11 +425,7 @@ array_resize(arrayobject *self, Py_ssize_t newsize) } if (data != NULL) { Py_ssize_t size = Py_SIZE(self); -#ifdef Py_GIL_DISABLED - atomic_itemcpy(newdata->items, data->items, Py_MIN(size, newsize), itemsize); -#else - memcpy(newdata->items, data->items, Py_MIN(size, newsize) * itemsize); -#endif + FT_ATOMIC_SOURCE_MEMCPY_RELAXED(newdata->items, data->items, Py_MIN(size, newsize) * itemsize); arraydata_free(data, _PyObject_GC_IS_SHARED(self)); } _Py_atomic_store_ptr_release(&self->data, newdata); @@ -1243,8 +1337,9 @@ array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) if (np == NULL) return NULL; if (ihigh > ilow) { - memcpy(np->data->items, a->data->items + ilow * a->ob_descr->itemsize, - (ihigh-ilow) * a->ob_descr->itemsize); + FT_ATOMIC_SOURCE_MEMCPY_RELAXED( + np->data->items, a->data->items + ilow * a->ob_descr->itemsize, + (ihigh-ilow) * a->ob_descr->itemsize); } return (PyObject *)np; } @@ -2895,7 +2990,7 @@ array_subscr_slice_lock_held(PyObject *op, PyObject *item) slicelength, self->ob_descr); if (result == NULL) return NULL; - memcpy(((arrayobject *)result)->data->items, + FT_ATOMIC_SOURCE_MEMCPY_RELAXED(((arrayobject *)result)->data->items, self->data->items + start * itemsize, slicelength * itemsize); return result; From 94ef41723ff1fb2288cb118bc900f5ef87a78255 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Tue, 11 Mar 2025 12:54:32 -0400 Subject: [PATCH 19/32] remove atomic memcpy --- Lib/test/test_array.py | 25 ++++---- Modules/arraymodule.c | 141 +---------------------------------------- 2 files changed, 16 insertions(+), 150 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index bb2203de627f52..5ad36c799e92c6 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1977,16 +1977,17 @@ def copy_random(b, a, count): for src, dst in idxs: a[dst] = a[src] - def extend_range(b, a, count): - b.wait() - for _ in range(count): - a.extend(range(10)) - - def extend_self(b, a, count): - c = a[:] - b.wait() - for _ in range(count): - a.extend(c) + # comment these back in if resize is made atomically quiet + # def extend_range(b, a, count): + # b.wait() + # for _ in range(count): + # a.extend(range(10)) + + # def extend_self(b, a, count): + # c = a[:] + # b.wait() + # for _ in range(count): + # a.extend(c) for tc in typecodes: if tc in 'uw': @@ -1996,10 +1997,10 @@ def extend_self(b, a, count): self.check([copy_back_and_forth] * 10, a, 100) self.check([append_and_pop] * 10, a, 100) - self.check([copy_back_and_forth] * 10 + [extend_self], a, 100) + # self.check([copy_back_and_forth] * 10 + [extend_self], a, 100) # comment this in if resize is made atomically quiet if tc not in 'uw': - self.check([copy_back_and_forth] * 10 + [extend_range], a, 100) + # self.check([copy_back_and_forth] * 10 + [extend_range], a, 100) # comment this in if resize is made atomically quiet self.check([copy_random] * 10, a * 5, 100) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index d348d313c8c798..966999789d6fb1 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -215,140 +215,6 @@ arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) #endif -#ifdef Py_GIL_DISABLED - -// This really doesn't belong here, for show at the moment. -static void -_Py_atomic_source_memcpy_relaxed(void *dest, void *src, size_t n) -{ - int diff = (int)((uintptr_t)dest ^ (uintptr_t)src); - - // the first half is needed to deal with misalignment - - if (diff & 1) { // dest and src not word aligned with each other - for (void *end = (char *)dest + n; dest < end; - dest = (char *)dest + 1, src = (char *)src + 1) { - *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); - } - - return; - } - - if ((uintptr_t)dest & 1) { // dest and src not word aligned in memory - if (n) { - *(char *)dest = _Py_atomic_load_char_relaxed((char *)src); - dest = (char *)dest + 1; - src = (char *)src + 1; - n -= 1; - } - - if (!n) { - return; - } - } - - if (diff & 2) { // dest and src not dword aligned with each other - size_t n2 = n / 2; - - for (void *end = (short *)dest + n2; dest < end; - dest = (short *)dest + 1, src = (short *)src + 1) { - *((short *)dest) = _Py_atomic_load_short_relaxed((short *)src); - } - - if (n & 1) { - *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); - } - - return; - } - - if ((uintptr_t)dest & 2) { // dest and src not dword aligned in memory - if (n >= 2) { - *(short *)dest = _Py_atomic_load_short_relaxed((short *)src); - dest = (short *)dest + 1; - src = (short *)src + 1; - n -= 2; - } - - if (!n) { - return; - } - } - - if (diff & 4) { // dest and src not qword aligned with each other - size_t n4 = n / 4; - - for (void *end = (PY_UINT32_T *)dest + n4; dest < end; - dest = (PY_UINT32_T *)dest + 1, src = (PY_UINT32_T *)src + 1) { - *((PY_UINT32_T *)dest) = (PY_UINT32_T)_Py_atomic_load_uint32_relaxed((PY_UINT32_T *)src); - } - - if (n & 2) { - *((short *)dest) = _Py_atomic_load_short_relaxed((short *)src); - dest = (short *)dest + 1; - src = (short *)src + 1; - } - - if (n & 1) { - *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); - } - - return; - } - - if ((uintptr_t)dest & 4) { // dest and src not qword aligned in memory - if (n >= 4) { - *(PY_UINT32_T *)dest = _Py_atomic_load_uint32_relaxed((PY_UINT32_T *)src); - dest = (PY_UINT32_T *)dest + 1; - src = (PY_UINT32_T *)src + 1; - n -= 4; - } - - if (!n) { - return; - } - } - - // the second half is aligned copy - - size_t n8 = n / 8; - - if (n8) { - for (void *end = (PY_UINT64_T *)dest + n8; dest < end; - dest = (PY_UINT64_T *)dest + 1, src = (PY_UINT64_T *)src + 1) { - *((PY_UINT64_T *)dest) = (PY_UINT64_T)_Py_atomic_load_uint64_relaxed((PY_UINT64_T *)src); - } - - n -= n8 * 8; - } - - if (n & 4) { - *((PY_UINT32_T *)dest) = (PY_UINT32_T)_Py_atomic_load_uint32_relaxed((PY_UINT32_T *)src); - dest = (PY_UINT32_T *)dest + 1; - src = (PY_UINT32_T *)src + 1; - } - - if (n & 2) { - *((short *)dest) = _Py_atomic_load_short_relaxed((short *)src); - dest = (short *)dest + 1; - src = (short *)src + 1; - } - - if (n & 1) { - *((char *)dest) = _Py_atomic_load_char_relaxed((char *)src); - } -} - -#define FT_ATOMIC_SOURCE_MEMCPY_RELAXED(dest, src, n) \ - _Py_atomic_source_memcpy_relaxed((dest), (src), (n)) - -#else - -#define FT_ATOMIC_SOURCE_MEMCPY_RELAXED(dest, src, n) \ - memcpy((dest), (src), (n)) - -#endif - static int array_resize(arrayobject *self, Py_ssize_t newsize) { @@ -425,7 +291,7 @@ array_resize(arrayobject *self, Py_ssize_t newsize) } if (data != NULL) { Py_ssize_t size = Py_SIZE(self); - FT_ATOMIC_SOURCE_MEMCPY_RELAXED(newdata->items, data->items, Py_MIN(size, newsize) * itemsize); + memcpy(newdata->items, data->items, Py_MIN(size, newsize) * itemsize); arraydata_free(data, _PyObject_GC_IS_SHARED(self)); } _Py_atomic_store_ptr_release(&self->data, newdata); @@ -1337,8 +1203,7 @@ array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) if (np == NULL) return NULL; if (ihigh > ilow) { - FT_ATOMIC_SOURCE_MEMCPY_RELAXED( - np->data->items, a->data->items + ilow * a->ob_descr->itemsize, + memcpy(np->data->items, a->data->items + ilow * a->ob_descr->itemsize, (ihigh-ilow) * a->ob_descr->itemsize); } return (PyObject *)np; @@ -2990,7 +2855,7 @@ array_subscr_slice_lock_held(PyObject *op, PyObject *item) slicelength, self->ob_descr); if (result == NULL) return NULL; - FT_ATOMIC_SOURCE_MEMCPY_RELAXED(((arrayobject *)result)->data->items, + memcpy(((arrayobject *)result)->data->items, self->data->items + start * itemsize, slicelength * itemsize); return result; From 143178432b11343154f1a491cd4439f515779d50 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Tue, 11 Mar 2025 13:11:20 -0400 Subject: [PATCH 20/32] regen clinic --- Modules/arraymodule.c | 10 +++++----- Modules/clinic/arraymodule.c.h | 11 ++++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 67d3660b75cd01..4aab445bbffabc 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -1252,7 +1252,7 @@ Return a copy of the array. static PyObject * array_array___deepcopy___impl(arrayobject *self, PyObject *unused) -/*[clinic end generated code: output=703b4c412feaaf31 input=2405ecb4933748c4]*/ +/*[clinic end generated code: output=703b4c412feaaf31 input=1a29f718f5b8a1dc]*/ { return array_array___copy___impl(self); } @@ -1528,7 +1528,7 @@ Return number of occurrences of v in the array. static PyObject * array_array_count_impl(arrayobject *self, PyObject *v) -/*[clinic end generated code: output=93ead26a2affb739 input=d9bce9d65e39d1f5]*/ +/*[clinic end generated code: output=93ead26a2affb739 input=c12c0042c1d0e27e]*/ { Py_ssize_t count = 0; Py_ssize_t i; @@ -1640,7 +1640,7 @@ Remove the first occurrence of v in the array. static PyObject * array_array_remove_impl(arrayobject *self, PyObject *v) -/*[clinic end generated code: output=f2a24e288ecb2a35 input=0b1e5aed25590027]*/ +/*[clinic end generated code: output=f2a24e288ecb2a35 input=78bef3fd40e62f7a]*/ { Py_ssize_t i; @@ -1794,7 +1794,7 @@ Append new value v to the end of the array. static PyObject * array_array_append_impl(arrayobject *self, PyObject *v) -/*[clinic end generated code: output=2f1e8cbad70c2a8b input=0b98d9d78e78f0fa]*/ +/*[clinic end generated code: output=2f1e8cbad70c2a8b input=9cdd897c66a40c3f]*/ { if (ins1(self, Py_SIZE(self), v) != 0) return NULL; @@ -2022,7 +2022,7 @@ Append items to array from list. static PyObject * array_array_fromlist_impl(arrayobject *self, PyObject *list) -/*[clinic end generated code: output=6c23733a68dd68df input=be2605a96c49680f]*/ +/*[clinic end generated code: output=6c23733a68dd68df input=c7c056aaf85d997a]*/ { Py_ssize_t n; diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 6d52ecdedf5098..569c116ca7266c 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -70,7 +70,7 @@ static PyObject * array_array___deepcopy___impl(arrayobject *self, PyObject *unused); static PyObject * -array_array___deepcopy__(arrayobject *self, PyObject *unused) +array_array___deepcopy__(PyObject *self, PyObject *unused) { PyObject *return_value = NULL; @@ -94,7 +94,7 @@ static PyObject * array_array_count_impl(arrayobject *self, PyObject *v); static PyObject * -array_array_count(arrayobject *self, PyObject *v) +array_array_count(PyObject *self, PyObject *v) { PyObject *return_value = NULL; @@ -166,7 +166,7 @@ static PyObject * array_array_remove_impl(arrayobject *self, PyObject *v); static PyObject * -array_array_remove(arrayobject *self, PyObject *v) +array_array_remove(PyObject *self, PyObject *v) { PyObject *return_value = NULL; @@ -351,7 +351,7 @@ static PyObject * array_array_append_impl(arrayobject *self, PyObject *v); static PyObject * -array_array_append(arrayobject *self, PyObject *v) +array_array_append(PyObject *self, PyObject *v) { PyObject *return_value = NULL; @@ -530,7 +530,7 @@ static PyObject * array_array_fromlist_impl(arrayobject *self, PyObject *list); static PyObject * -array_array_fromlist(arrayobject *self, PyObject *list) +array_array_fromlist(PyObject *self, PyObject *list) { PyObject *return_value = NULL; @@ -846,3 +846,4 @@ array_arrayiterator___setstate__(PyObject *self, PyObject *state) return return_value; } +/*[clinic end generated code: output=46ca729af0fdb7cd input=a9049054013a1b77]*/ From 409239c9fe4cf43d0756163efc59e2257ac98e4f Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sat, 15 Mar 2025 13:31:02 -0400 Subject: [PATCH 21/32] requested changes --- Lib/test/test_array.py | 50 ----------- Modules/arraymodule.c | 194 ++++++++++++++++++----------------------- 2 files changed, 87 insertions(+), 157 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 5ad36c799e92c6..75c4b10adbbe02 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1953,56 +1953,6 @@ def iter_reduce(b, a, it): # iterator stuff self.check([clear] + [iter_next] * 10, a := array.array('i', [1] * 10), iter(a)) - @unittest.skipUnless(support.check_sanitizer(thread=True), 'meant for tsan') - @threading_helper.reap_threads - @threading_helper.requires_working_threading() - def test_free_threading_tsan(self): - def copy_back_and_forth(b, a, count): - b.wait() - for _ in range(count): - a[0] = a[1] - a[1] = a[0] - - def append_and_pop(b, a, count): - v = a[0] - b.wait() - for _ in range(count): - a.append(v) - a.pop() - - def copy_random(b, a, count): - end = len(a) - 1 - idxs = [(random.randint(0, end), random.randint(0, end)) for _ in range(count)] - b.wait() - for src, dst in idxs: - a[dst] = a[src] - - # comment these back in if resize is made atomically quiet - # def extend_range(b, a, count): - # b.wait() - # for _ in range(count): - # a.extend(range(10)) - - # def extend_self(b, a, count): - # c = a[:] - # b.wait() - # for _ in range(count): - # a.extend(c) - - for tc in typecodes: - if tc in 'uw': - a = array.array(tc, 'ab') - else: - a = array.array(tc, [0, 1]) - - self.check([copy_back_and_forth] * 10, a, 100) - self.check([append_and_pop] * 10, a, 100) - # self.check([copy_back_and_forth] * 10 + [extend_self], a, 100) # comment this in if resize is made atomically quiet - - if tc not in 'uw': - # self.check([copy_back_and_forth] * 10 + [extend_range], a, 100) # comment this in if resize is made atomically quiet - self.check([copy_random] * 10, a * 5, 100) - if __name__ == "__main__": unittest.main() diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 4aab445bbffabc..c3921575e4765f 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -73,8 +73,8 @@ typedef struct { PyObject *str_iter; } array_state; -static inline -Py_ssize_t PyArray_GET_SIZE(PyObject *op) { +static inline Py_ssize_t +PyArray_GET_SIZE(PyObject *op) { arrayobject *ao = (arrayobject *)op; #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&(_PyVarObject_CAST(ao)->ob_size)); @@ -149,16 +149,10 @@ enum machine_format_code { #define array_Check(op, state) PyObject_TypeCheck(op, state->ArrayType) -static inline char * -arraydata_safeitems(arraydata *data) -{ - return data == NULL ? NULL : data->items; -} - static arraydata * arraydata_alloc(Py_ssize_t size, int itemsize) { - if (size > PY_SSIZE_T_MAX/itemsize - 1) { + if ((size_t) size > (PY_SSIZE_T_MAX - sizeof(arraydata))/itemsize - 1) { return NULL; } arraydata *data = (arraydata *)PyMem_Malloc(sizeof(arraydata) + size * itemsize); @@ -202,7 +196,7 @@ arraydata_set_items(arraydata *data, char *newitems, Py_ssize_t newsize, int ite static arraydata * arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) { - if (size > PY_SSIZE_T_MAX/itemsize - 1) { + if ((size_t) size > (PY_SSIZE_T_MAX - sizeof(arraydata))/itemsize - 1) { return NULL; } data = (arraydata *)PyMem_Realloc(data, sizeof(arraydata) + size * itemsize); @@ -215,6 +209,12 @@ arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) #endif +static inline char * +array_items_ptr(arrayobject *self) +{ + return self->data == NULL ? NULL : self->data->items; +} + static int array_resize(arrayobject *self, Py_ssize_t newsize) { @@ -323,8 +323,8 @@ in bounds; that's the responsibility of the caller. static PyObject * b_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong( - (long) (signed char) FT_ATOMIC_LOAD_CHAR_RELAXED(items[i])); + long x = ((signed char *)items)[i]; + return PyLong_FromLong(x); } static int @@ -348,7 +348,7 @@ b_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_CHAR_RELAXED(items[i], (char) x); + ((char *)items)[i] = (char)x; } return 0; } @@ -356,8 +356,8 @@ b_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * BB_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong( - (long) FT_ATOMIC_LOAD_UCHAR_RELAXED(((unsigned char *) items)[i])); + long x = ((unsigned char *)items)[i]; + return PyLong_FromLong(x); } static int @@ -369,7 +369,7 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_UCHAR_RELAXED(((unsigned char *) items)[i], x); + ((unsigned char *)items)[i] = x; } return 0; } @@ -377,13 +377,7 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * u_getitem(char *items, Py_ssize_t i) { -#if SIZEOF_WCHAR_T > 2 - return PyUnicode_FromOrdinal( - (wchar_t) FT_ATOMIC_LOAD_INT_RELAXED(((wchar_t *) items)[i])); -#else - return PyUnicode_FromOrdinal( - (wchar_t) FT_ATOMIC_LOAD_SHORT_RELAXED(((wchar_t *) items)[i])); -#endif + return PyUnicode_FromOrdinal(((wchar_t *) items)[i]); } static int @@ -418,11 +412,7 @@ u_setitem(char *items, Py_ssize_t i, PyObject *v) assert(len == 1); if (i >= 0) { -#if SIZEOF_WCHAR_T > 2 - FT_ATOMIC_STORE_INT_RELAXED(((wchar_t *) items)[i], w); -#else - FT_ATOMIC_STORE_SHORT_RELAXED(((wchar_t *) items)[i], w); -#endif + ((wchar_t *)items)[i] = w; } return 0; } @@ -430,8 +420,7 @@ u_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * w_getitem(char *items, Py_ssize_t i) { - return PyUnicode_FromOrdinal( - (Py_UCS4) FT_ATOMIC_LOAD_UINT32_RELAXED(((Py_UCS4 *) items)[i])); + return PyUnicode_FromOrdinal(((Py_UCS4 *) items)[i]); } static int @@ -453,8 +442,7 @@ w_setitem(char *items, Py_ssize_t i, PyObject *v) } if (i >= 0) { - Py_UCS4 w = PyUnicode_READ_CHAR(v, 0); - FT_ATOMIC_STORE_UINT32_RELAXED(((Py_UCS4 *) items)[i], w); + ((Py_UCS4 *)items)[i] = PyUnicode_READ_CHAR(v, 0); } return 0; } @@ -462,8 +450,7 @@ w_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * h_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong( - (long) FT_ATOMIC_LOAD_SHORT_RELAXED(((short *) items)[i])); + return PyLong_FromLong((long) ((short *)items)[i]); } @@ -476,7 +463,7 @@ h_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_SHORT_RELAXED(((short *) items)[i], x); + ((short *)items)[i] = x; } return 0; } @@ -484,8 +471,7 @@ h_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * HH_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong( - (long) FT_ATOMIC_LOAD_USHORT_RELAXED(((unsigned short *) items)[i])); + return PyLong_FromLong((long) ((unsigned short *)items)[i]); } static int @@ -508,7 +494,7 @@ HH_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_USHORT_RELAXED(((unsigned short *) items)[i], (unsigned short) x); + ((short *)items)[i] = (short)x; } return 0; } @@ -516,8 +502,7 @@ HH_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * i_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong( - (long) FT_ATOMIC_LOAD_INT_RELAXED(((int *) items)[i])); + return PyLong_FromLong((long) ((int *)items)[i]); } static int @@ -529,7 +514,7 @@ i_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_INT_RELAXED(((int *)items)[i], x); + ((int *)items)[i] = x; } return 0; } @@ -538,7 +523,7 @@ static PyObject * II_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLong( - (unsigned long) FT_ATOMIC_LOAD_UINT_RELAXED(((unsigned int *) items)[i])); + (unsigned long) ((unsigned int *)items)[i]); } static int @@ -570,7 +555,7 @@ II_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_UINT_RELAXED(((unsigned int *) items)[i], (unsigned int) x); + ((unsigned int *)items)[i] = (unsigned int)x; } if (do_decref) { @@ -582,8 +567,7 @@ II_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * l_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLong( - FT_ATOMIC_LOAD_LONG_RELAXED(((long *) items)[i])); + return PyLong_FromLong(((long *)items)[i]); } static int @@ -594,7 +578,7 @@ l_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_LONG_RELAXED(((long *) items)[i], x); + ((long *)items)[i] = x; } return 0; } @@ -602,8 +586,7 @@ l_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * LL_getitem(char *items, Py_ssize_t i) { - return PyLong_FromUnsignedLong( - FT_ATOMIC_LOAD_ULONG_RELAXED(((unsigned long *) items)[i])); + return PyLong_FromUnsignedLong(((unsigned long *)items)[i]); } static int @@ -627,7 +610,7 @@ LL_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_ULONG_RELAXED(((unsigned long *) items)[i], x); + ((unsigned long *)items)[i] = x; } if (do_decref) { @@ -639,8 +622,7 @@ LL_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * q_getitem(char *items, Py_ssize_t i) { - return PyLong_FromLongLong( - FT_ATOMIC_LOAD_LLONG_RELAXED(((long long *) items)[i])); + return PyLong_FromLongLong(((long long *)items)[i]); } static int @@ -651,7 +633,7 @@ q_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_LLONG_RELAXED(((long long *) items)[i], x); + ((long long *)items)[i] = x; } return 0; } @@ -660,7 +642,7 @@ static PyObject * QQ_getitem(char *items, Py_ssize_t i) { return PyLong_FromUnsignedLongLong( - FT_ATOMIC_LOAD_ULLONG_RELAXED(((unsigned long long *) items)[i])); + ((unsigned long long *)items)[i]); } static int @@ -684,7 +666,7 @@ QQ_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_ULLONG_RELAXED(((unsigned long long *) items)[i], x); + ((unsigned long long *)items)[i] = x; } if (do_decref) { @@ -696,8 +678,7 @@ QQ_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * f_getitem(char *items, Py_ssize_t i) { - return PyFloat_FromDouble( - (double) FT_ATOMIC_LOAD_FLOAT_RELAXED(((float *) items)[i])); + return PyFloat_FromDouble((double) ((float *)items)[i]); } static int @@ -708,7 +689,7 @@ f_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_FLOAT_RELAXED(((float *) items)[i], x); + ((float *)items)[i] = x; } return 0; } @@ -716,8 +697,7 @@ f_setitem(char *items, Py_ssize_t i, PyObject *v) static PyObject * d_getitem(char *items, Py_ssize_t i) { - return PyFloat_FromDouble( - FT_ATOMIC_LOAD_DOUBLE_RELAXED(((double *) items)[i])); + return PyFloat_FromDouble(((double *)items)[i]); } static int @@ -728,7 +708,7 @@ d_setitem(char *items, Py_ssize_t i, PyObject *v) return -1; } if (i >= 0) { - FT_ATOMIC_STORE_DOUBLE_RELAXED(((double *) items)[i], x); + ((double *)items)[i] = x; } return 0; } @@ -994,7 +974,7 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) if (array_resize(self, n+1) == -1) return -1; - items = self->data->items; + items = array_items_ptr(self); if (where < 0) { where += n; if (where < 0) @@ -1072,7 +1052,7 @@ array_richcompare_lock_held(PyObject *v, PyObject *w, int op) /* Fast path: arrays with same types can have their buffers compared directly */ Py_ssize_t common_length = Py_MIN(Py_SIZE(va), Py_SIZE(wa)); - int result = va->ob_descr->compareitems(va->data->items, wa->data->items, + int result = va->ob_descr->compareitems(array_items_ptr(va), array_items_ptr(wa), common_length); if (result == 0) goto compare_sizes; @@ -1095,11 +1075,11 @@ array_richcompare_lock_held(PyObject *v, PyObject *w, int op) /* Search for the first index where items are different */ k = 1; for (i = 0; i < Py_SIZE(va) && i < Py_SIZE(wa); i++) { - vi = getarrayitem(va, i, va->data->items); + vi = getarrayitem(va, i, array_items_ptr(va)); if (vi == NULL) { return NULL; } - wi = getarrayitem(wa, i, wa->data->items); + wi = getarrayitem(wa, i, array_items_ptr(wa)); if (wi == NULL) { Py_DECREF(vi); return NULL; @@ -1203,7 +1183,7 @@ array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) if (np == NULL) return NULL; if (ihigh > ilow) { - memcpy(np->data->items, a->data->items + ilow * a->ob_descr->itemsize, + memcpy(array_items_ptr(np), array_items_ptr(a) + ilow * a->ob_descr->itemsize, (ihigh-ilow) * a->ob_descr->itemsize); } return (PyObject *)np; @@ -1286,11 +1266,11 @@ array_concat_lock_held(PyObject *op, PyObject *bb) return NULL; } if (Py_SIZE(a) > 0) { - memcpy(np->data->items, a->data->items, Py_SIZE(a)*a->ob_descr->itemsize); + memcpy(array_items_ptr(np), array_items_ptr(a), Py_SIZE(a)*a->ob_descr->itemsize); } if (Py_SIZE(b) > 0) { - memcpy(np->data->items + Py_SIZE(a)*a->ob_descr->itemsize, - b->data->items, Py_SIZE(b)*b->ob_descr->itemsize); + memcpy(array_items_ptr(np) + Py_SIZE(a)*a->ob_descr->itemsize, + array_items_ptr(b), Py_SIZE(b)*b->ob_descr->itemsize); } return (PyObject *)np; #undef b @@ -1328,7 +1308,7 @@ array_repeat_lock_held(PyObject *op, Py_ssize_t n) const Py_ssize_t oldbytes = array_length * a->ob_descr->itemsize; const Py_ssize_t newbytes = oldbytes * n; - _PyBytes_Repeat(np->data->items, newbytes, a->data->items, oldbytes); + _PyBytes_Repeat(array_items_ptr(np), newbytes, array_items_ptr(a), oldbytes); return (PyObject *)np; } @@ -1359,7 +1339,7 @@ array_del_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) ihigh = ilow; else if (ihigh > Py_SIZE(a)) ihigh = Py_SIZE(a); - item = a->data->items; + item = array_items_ptr(a); d = ihigh-ilow; /* Issue #4509: If the array has exported buffers and the slice assignment would change the size of the array, fail early to make @@ -1444,8 +1424,8 @@ array_do_extend_lock_held(array_state *state, arrayobject *self, PyObject *bb) if (array_resize(self, size) == -1) return -1; if (bbsize > 0) { - memcpy(self->data->items + oldsize * self->ob_descr->itemsize, - b->data->items, bbsize * b->ob_descr->itemsize); + memcpy(array_items_ptr(self) + oldsize * self->ob_descr->itemsize, + array_items_ptr(b), bbsize * b->ob_descr->itemsize); } return 0; @@ -1500,7 +1480,7 @@ array_inplace_repeat_lock_held(PyObject *op, Py_ssize_t n) if (array_resize(self, n * array_size) == -1) return NULL; - _PyBytes_Repeat(self->data->items, n*size, self->data->items, size); + _PyBytes_Repeat(array_items_ptr(self), n*size, array_items_ptr(self), size); } return Py_NewRef(self); } @@ -1537,7 +1517,7 @@ array_array_count_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem(self, i, self->data->items); + selfi = getarrayitem(self, i, array_items_ptr(self)); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1585,7 +1565,7 @@ array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, PyObject *selfi; int cmp; - selfi = getarrayitem(self, i, self->data->items); + selfi = getarrayitem(self, i, array_items_ptr(self)); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1648,7 +1628,7 @@ array_array_remove_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem(self, i, self->data->items); + selfi = getarrayitem(self, i, array_items_ptr(self)); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1694,7 +1674,7 @@ array_array_pop_impl(arrayobject *self, Py_ssize_t i) PyErr_SetString(PyExc_IndexError, "pop index out of range"); return NULL; } - v = getarrayitem(self, i, self->data->items); + v = getarrayitem(self, i, array_items_ptr(self)); if (v == NULL) return NULL; if (array_del_slice(self, i, i+1) != 0) { @@ -1765,7 +1745,7 @@ array_array_buffer_info_impl(arrayobject *self) if (!retval) return NULL; - v = PyLong_FromVoidPtr(arraydata_safeitems(self->data)); + v = PyLong_FromVoidPtr(array_items_ptr(self)); if (v == NULL) { Py_DECREF(retval); return NULL; @@ -1822,14 +1802,14 @@ array_array_byteswap_impl(arrayobject *self) case 1: break; case 2: - for (p = self->data->items, i = Py_SIZE(self); --i >= 0; p += 2) { + for (p = array_items_ptr(self), i = Py_SIZE(self); --i >= 0; p += 2) { char p0 = p[0]; p[0] = p[1]; p[1] = p0; } break; case 4: - for (p = self->data->items, i = Py_SIZE(self); --i >= 0; p += 4) { + for (p = array_items_ptr(self), i = Py_SIZE(self); --i >= 0; p += 4) { char p0 = p[0]; char p1 = p[1]; p[0] = p[3]; @@ -1839,7 +1819,7 @@ array_array_byteswap_impl(arrayobject *self) } break; case 8: - for (p = self->data->items, i = Py_SIZE(self); --i >= 0; p += 8) { + for (p = array_items_ptr(self), i = Py_SIZE(self); --i >= 0; p += 8) { char p0 = p[0]; char p1 = p[1]; char p2 = p[2]; @@ -1880,8 +1860,8 @@ array_array_reverse_impl(arrayobject *self) assert((size_t)itemsize <= sizeof(tmp)); if (Py_SIZE(self) > 1) { - for (p = self->data->items, - q = self->data->items + (Py_SIZE(self) - 1)*itemsize; + for (p = array_items_ptr(self), + q = array_items_ptr(self) + (Py_SIZE(self) - 1)*itemsize; p < q; p += itemsize, q -= itemsize) { /* memory areas guaranteed disjoint, so memcpy @@ -1990,7 +1970,7 @@ array_array_tofile_impl(arrayobject *self, PyTypeObject *cls, PyObject *f) assert(state != NULL); for (i = 0; i < nblocks; i++) { - char *ptr = self->data->items + i*BLOCKSIZE; + char *ptr = array_items_ptr(self) + i*BLOCKSIZE; Py_ssize_t size = BLOCKSIZE; PyObject *bytes, *res; @@ -2038,7 +2018,7 @@ array_array_fromlist_impl(arrayobject *self, PyObject *list) return NULL; for (i = 0; i < n; i++) { PyObject *v = PyList_GET_ITEM(list, i); - if (setarrayitem(self, Py_SIZE(self) - n + i, v, self->data->items) != 0) { + if (setarrayitem(self, Py_SIZE(self) - n + i, v, array_items_ptr(self)) != 0) { array_resize(self, old_size); return NULL; } @@ -2070,7 +2050,7 @@ array_array_tolist_impl(arrayobject *self) if (list == NULL) return NULL; for (i = 0; i < Py_SIZE(self); i++) { - PyObject *v = getarrayitem(self, i, self->data->items); + PyObject *v = getarrayitem(self, i, array_items_ptr(self)); if (v == NULL) goto error; PyList_SET_ITEM(list, i, v); @@ -2119,7 +2099,7 @@ array_array_frombytes_impl(arrayobject *self, Py_buffer *buffer) if (array_resize(self, old_size + n) == -1) { return NULL; } - memcpy(self->data->items + old_size * itemsize, + memcpy(array_items_ptr(self) + old_size * itemsize, buffer->buf, n * itemsize); } Py_RETURN_NONE; @@ -2137,7 +2117,7 @@ array_array_tobytes_impl(arrayobject *self) /*[clinic end generated code: output=87318e4edcdc2bb6 input=c4d44d5499d2320f]*/ { if (Py_SIZE(self) <= PY_SSIZE_T_MAX / self->ob_descr->itemsize) { - return PyBytes_FromStringAndSize(arraydata_safeitems(self->data), + return PyBytes_FromStringAndSize(array_items_ptr(self), Py_SIZE(self) * self->ob_descr->itemsize); } else { return PyErr_NoMemory(); @@ -2182,7 +2162,7 @@ array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) // must not fail PyUnicode_AsWideChar( - ustr, ((wchar_t *)arraydata_safeitems(self->data)) + old_size, ustr_length); + ustr, ((wchar_t *)array_items_ptr(self)) + old_size, ustr_length); } } else { // typecode == 'w' @@ -2198,7 +2178,7 @@ array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) } // must not fail - Py_UCS4 *u = PyUnicode_AsUCS4(ustr, ((Py_UCS4*)arraydata_safeitems(self->data)) + old_size, + Py_UCS4 *u = PyUnicode_AsUCS4(ustr, ((Py_UCS4*)array_items_ptr(self)) + old_size, ustr_length, 0); assert(u != NULL); (void)u; // Suppress unused_variable warning. @@ -2229,11 +2209,11 @@ array_array_tounicode_impl(arrayobject *self) return NULL; } if (typecode == 'u') { - return PyUnicode_FromWideChar((wchar_t *) arraydata_safeitems(self->data), Py_SIZE(self)); + return PyUnicode_FromWideChar((wchar_t *) array_items_ptr(self), Py_SIZE(self)); } else { // typecode == 'w' int byteorder = 0; // native byteorder - return PyUnicode_DecodeUTF32((const char *) arraydata_safeitems(self->data), Py_SIZE(self) * 4, + return PyUnicode_DecodeUTF32((const char *) array_items_ptr(self), Py_SIZE(self) * 4, NULL, &byteorder); } } @@ -2856,7 +2836,7 @@ array_subscr_slice_lock_held(PyObject *op, PyObject *item) if (result == NULL) return NULL; memcpy(((arrayobject *)result)->data->items, - self->data->items + start * itemsize, + array_items_ptr(self) + start * itemsize, slicelength * itemsize); return result; } @@ -2869,7 +2849,7 @@ array_subscr_slice_lock_held(PyObject *op, PyObject *item) for (cur = start, i = 0; i < slicelength; cur += step, i++) { memcpy(ar->data->items + i*itemsize, - self->data->items + cur*itemsize, + array_items_ptr(self) + cur*itemsize, itemsize); } @@ -2940,7 +2920,7 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) slicelength = 1; } else - return setarrayitem(self, i, value, self->data->items); + return setarrayitem(self, i, value, array_items_ptr(self)); } else if (PySlice_Check(item)) { if (PySlice_Unpack(item, &start, &stop, &step) < 0) { @@ -2999,8 +2979,8 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) if (step == 1) { if (slicelength > needed) { - memmove(self->data->items + (start + needed) * itemsize, - self->data->items + stop * itemsize, + memmove(array_items_ptr(self) + (start + needed) * itemsize, + array_items_ptr(self) + stop * itemsize, (Py_SIZE(self) - stop) * itemsize); if (array_resize(self, Py_SIZE(self) + needed - slicelength) < 0) @@ -3010,12 +2990,12 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) if (array_resize(self, Py_SIZE(self) + needed - slicelength) < 0) return -1; - memmove(self->data->items + (start + needed) * itemsize, - self->data->items + stop * itemsize, + memmove(array_items_ptr(self) + (start + needed) * itemsize, + array_items_ptr(self) + stop * itemsize, (Py_SIZE(self) - start - needed) * itemsize); } if (needed > 0) - memcpy(self->data->items + start * itemsize, + memcpy(array_items_ptr(self) + start * itemsize, other->data->items, needed * itemsize); return 0; } @@ -3035,14 +3015,14 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) if (cur + step >= (size_t)Py_SIZE(self)) lim = Py_SIZE(self) - cur - 1; - memmove(self->data->items + (cur - i) * itemsize, - self->data->items + (cur + 1) * itemsize, + memmove(array_items_ptr(self) + (cur - i) * itemsize, + array_items_ptr(self) + (cur + 1) * itemsize, lim * itemsize); } cur = start + (size_t)slicelength * step; if (cur < (size_t)Py_SIZE(self)) { - memmove(self->data->items + (cur-slicelength) * itemsize, - self->data->items + cur * itemsize, + memmove(array_items_ptr(self) + (cur-slicelength) * itemsize, + array_items_ptr(self) + cur * itemsize, (Py_SIZE(self) - cur) * itemsize); } if (array_resize(self, Py_SIZE(self) - slicelength) < 0) @@ -3062,7 +3042,7 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) } for (cur = start, i = 0; i < slicelength; cur += step, i++) { - memcpy(self->data->items + cur * itemsize, + memcpy(array_items_ptr(self) + cur * itemsize, other->data->items + i * itemsize, itemsize); } @@ -3113,7 +3093,7 @@ array_buffer_getbuf_lock_held(PyObject *op, Py_buffer *view, int flags) return -1; } - view->buf = (void *)arraydata_safeitems(self->data); + view->buf = (void *)array_items_ptr(self); view->obj = Py_NewRef(self); if (view->buf == NULL) view->buf = (void *)emptybuf; @@ -3285,7 +3265,7 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) else if (initial != NULL && array_Check(initial, state) && len > 0) { arrayobject *self = (arrayobject *)a; arrayobject *other = (arrayobject *)initial; - memcpy(self->data->items, other->data->items, len * other->ob_descr->itemsize); + memcpy(array_items_ptr(self), other->data->items, len * other->ob_descr->itemsize); } if (it != NULL) { if (array_iter_extend((arrayobject *)a, it) == -1) { From a6f17c90fc4b1d6a51192e30678a76cdc03d61e2 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 17 Mar 2025 18:15:27 -0400 Subject: [PATCH 22/32] more requested changes --- Modules/arraymodule.c | 116 ++++++++++++++++++------------------------ 1 file changed, 49 insertions(+), 67 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index c3921575e4765f..f33f6d4fea186c 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -44,7 +44,7 @@ struct arraydescr { typedef struct { Py_ssize_t allocated; - char items[]; + _Alignas(8) char items[]; } arraydata; typedef struct arrayobject { @@ -73,7 +73,7 @@ typedef struct { PyObject *str_iter; } array_state; -static inline Py_ssize_t +static Py_ssize_t PyArray_GET_SIZE(PyObject *op) { arrayobject *ao = (arrayobject *)op; #ifdef Py_GIL_DISABLED @@ -152,9 +152,6 @@ enum machine_format_code { static arraydata * arraydata_alloc(Py_ssize_t size, int itemsize) { - if ((size_t) size > (PY_SSIZE_T_MAX - sizeof(arraydata))/itemsize - 1) { - return NULL; - } arraydata *data = (arraydata *)PyMem_Malloc(sizeof(arraydata) + size * itemsize); if (data == NULL) { return NULL; @@ -180,25 +177,11 @@ arraydata_free(arraydata *data, bool use_qsbr) #endif } -static arraydata * -arraydata_set_items(arraydata *data, char *newitems, Py_ssize_t newsize, int itemsize, bool use_qsbr) -{ - arraydata_free(data, use_qsbr); - data = arraydata_alloc(newsize, itemsize); - if (data != NULL) { - memcpy(data->items, newitems, newsize * itemsize); - } - return data; -} - #ifndef Py_GIL_DISABLED static arraydata * arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) { - if ((size_t) size > (PY_SSIZE_T_MAX - sizeof(arraydata))/itemsize - 1) { - return NULL; - } data = (arraydata *)PyMem_Realloc(data, sizeof(arraydata) + size * itemsize); if (data == NULL) { return NULL; @@ -209,7 +192,7 @@ arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) #endif -static inline char * +static char * array_items_ptr(arrayobject *self) { return self->data == NULL ? NULL : self->data->items; @@ -805,25 +788,25 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *des return (PyObject *) op; } -static inline int +static int valid_index(Py_ssize_t i, Py_ssize_t limit) { return (size_t) i < (size_t) limit; } -static inline PyObject * -getarrayitem(arrayobject *ap, Py_ssize_t i, char *items) +static PyObject * +getarrayitem(arrayobject *ap, Py_ssize_t i, arraydata *data) { #ifndef NDEBUG array_state *state = find_array_state_by_type(Py_TYPE(ap)); assert(array_Check(ap, state)); #ifdef Py_GIL_DISABLED - assert(valid_index(i, (_Py_CONTAINER_OF(items, arraydata, items))->allocated)); + assert(valid_index(i, data->allocated)); #else assert(valid_index(i, Py_SIZE(ap))); #endif #endif - return (*ap->ob_descr->getitem)(items, i); + return (*ap->ob_descr->getitem)(data->items, i); } static PyObject * @@ -841,7 +824,7 @@ getarrayitem_locked(PyObject *op, Py_ssize_t i) } else { arrayobject *ap = (arrayobject *)op; - ret = getarrayitem(ap, i, ap->data->items); + ret = getarrayitem(ap, i, ap->data); } Py_END_CRITICAL_SECTION(); return ret; @@ -850,7 +833,7 @@ getarrayitem_locked(PyObject *op, Py_ssize_t i) #ifdef Py_GIL_DISABLED static PyObject * -getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) +getarrayitem_threadsafe(PyObject *op, Py_ssize_t i) { if (!_Py_IsOwnedByCurrentThread((PyObject *)op) && !_PyObject_GC_IS_SHARED(op)) { return getarrayitem_locked(op, i); @@ -867,37 +850,34 @@ getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) if (!valid_index(i, data->allocated)) { return NULL; } - return getarrayitem(ap, i, data->items); - /* Could check size again here to make sure it hasn't changed but since - there isn't a well defined order between unsynchronized thread - operations there's no point. */ + return getarrayitem(ap, i, data); } #else // Py_GIL_DISABLED static PyObject * -getarrayitem_maybe_locked(PyObject *op, Py_ssize_t i) +getarrayitem_threadsafe(PyObject *op, Py_ssize_t i) { return getarrayitem_locked(op, i); } #endif // Py_GIL_DISABLED -static inline int -setarrayitem(arrayobject *ap, Py_ssize_t i, PyObject *v, char *items) +static int +setarrayitem(arrayobject *ap, Py_ssize_t i, PyObject *v, arraydata *data) { #ifndef NDEBUG array_state *state = find_array_state_by_type(Py_TYPE(ap)); assert(array_Check(ap, state)); - if (items != NULL) { + if (data != NULL) { #ifdef Py_GIL_DISABLED - assert(valid_index(i, (_Py_CONTAINER_OF(items, arraydata, items))->allocated)); + assert(valid_index(i, data->allocated)); #else assert(valid_index(i, Py_SIZE(ap))); #endif } #endif - return (*ap->ob_descr->setitem)(items, i, v); + return (*ap->ob_descr->setitem)(data->items, i, v); } static int @@ -916,7 +896,7 @@ setarrayitem_locked(PyObject *op, Py_ssize_t i, PyObject *v) } else { arrayobject *ap = arrayobject_CAST(op); - ret = setarrayitem(ap, i, v, ap->data->items); + ret = setarrayitem(ap, i, v, ap->data); } Py_END_CRITICAL_SECTION(); return ret; @@ -925,7 +905,7 @@ setarrayitem_locked(PyObject *op, Py_ssize_t i, PyObject *v) #ifdef Py_GIL_DISABLED static int -setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) +setarrayitem_threadsafe(PyObject *op, Py_ssize_t i, PyObject *v) { if (!_Py_IsOwnedByCurrentThread((PyObject *)op) && !_PyObject_GC_IS_SHARED(op)) { return setarrayitem_locked(op, i, v); @@ -942,7 +922,7 @@ setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) if (!valid_index(i, data->allocated)) { goto error; } - return setarrayitem(ap, i, v, data->items); + return setarrayitem(ap, i, v, data); error: PyErr_SetString(PyExc_IndexError, "array index out of range"); @@ -952,7 +932,7 @@ setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) #else // Py_GIL_DISABLED static int -setarrayitem_maybe_locked(PyObject *op, Py_ssize_t i, PyObject *v) +setarrayitem_threadsafe(PyObject *op, Py_ssize_t i, PyObject *v) { return setarrayitem_locked(op, i, v); } @@ -963,7 +943,6 @@ static int ins1(arrayobject *self, Py_ssize_t where, PyObject *v) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); - char *items; Py_ssize_t n = Py_SIZE(self); if (v == NULL) { PyErr_BadInternalCall(); @@ -974,7 +953,6 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) if (array_resize(self, n+1) == -1) return -1; - items = array_items_ptr(self); if (where < 0) { where += n; if (where < 0) @@ -984,10 +962,10 @@ ins1(arrayobject *self, Py_ssize_t where, PyObject *v) where = n; /* appends don't need to call memmove() */ if (where != n) - memmove(items + (where+1)*self->ob_descr->itemsize, - items + where*self->ob_descr->itemsize, + memmove(array_items_ptr(self) + (where+1)*self->ob_descr->itemsize, + array_items_ptr(self) + where*self->ob_descr->itemsize, (n-where)*self->ob_descr->itemsize); - return setarrayitem(self, where, v, items); + return setarrayitem(self, where, v, self->data); } /* Methods */ @@ -1075,11 +1053,11 @@ array_richcompare_lock_held(PyObject *v, PyObject *w, int op) /* Search for the first index where items are different */ k = 1; for (i = 0; i < Py_SIZE(va) && i < Py_SIZE(wa); i++) { - vi = getarrayitem(va, i, array_items_ptr(va)); + vi = getarrayitem(va, i, va->data); if (vi == NULL) { return NULL; } - wi = getarrayitem(wa, i, array_items_ptr(wa)); + wi = getarrayitem(wa, i, wa->data); if (wi == NULL) { Py_DECREF(vi); return NULL; @@ -1154,7 +1132,7 @@ array_length(PyObject *op) static PyObject * array_item(PyObject *op, Py_ssize_t i) { - PyObject *item = getarrayitem_maybe_locked(op, i); + PyObject *item = getarrayitem_threadsafe(op, i); if (item == NULL) { PyErr_SetString(PyExc_IndexError, "array index out of range"); } @@ -1363,7 +1341,7 @@ static int array_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) { if (v != NULL) { - return setarrayitem_maybe_locked(op, i, v); + return setarrayitem_threadsafe(op, i, v); } int ret; Py_BEGIN_CRITICAL_SECTION(op); @@ -1517,7 +1495,7 @@ array_array_count_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem(self, i, array_items_ptr(self)); + selfi = getarrayitem(self, i, self->data); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1565,7 +1543,7 @@ array_array_index_impl(arrayobject *self, PyObject *v, Py_ssize_t start, PyObject *selfi; int cmp; - selfi = getarrayitem(self, i, array_items_ptr(self)); + selfi = getarrayitem(self, i, self->data); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1589,7 +1567,7 @@ array_contains_lock_held(PyObject *op, PyObject *v) for (i = 0, cmp = 0 ; cmp == 0 && i < Py_SIZE(op); i++) { arrayobject *ap = (arrayobject *)op; - PyObject *opi = getarrayitem(ap, i, ap->data->items); + PyObject *opi = getarrayitem(ap, i, ap->data); if (opi == NULL) return -1; cmp = PyObject_RichCompareBool(opi, v, Py_EQ); @@ -1628,7 +1606,7 @@ array_array_remove_impl(arrayobject *self, PyObject *v) PyObject *selfi; int cmp; - selfi = getarrayitem(self, i, array_items_ptr(self)); + selfi = getarrayitem(self, i, self->data); if (selfi == NULL) return NULL; cmp = PyObject_RichCompareBool(selfi, v, Py_EQ); @@ -1674,7 +1652,7 @@ array_array_pop_impl(arrayobject *self, Py_ssize_t i) PyErr_SetString(PyExc_IndexError, "pop index out of range"); return NULL; } - v = getarrayitem(self, i, array_items_ptr(self)); + v = getarrayitem(self, i, self->data); if (v == NULL) return NULL; if (array_del_slice(self, i, i+1) != 0) { @@ -2018,7 +1996,7 @@ array_array_fromlist_impl(arrayobject *self, PyObject *list) return NULL; for (i = 0; i < n; i++) { PyObject *v = PyList_GET_ITEM(list, i); - if (setarrayitem(self, Py_SIZE(self) - n + i, v, array_items_ptr(self)) != 0) { + if (setarrayitem(self, Py_SIZE(self) - n + i, v, self->data) != 0) { array_resize(self, old_size); return NULL; } @@ -2050,7 +2028,7 @@ array_array_tolist_impl(arrayobject *self) if (list == NULL) return NULL; for (i = 0; i < Py_SIZE(self); i++) { - PyObject *v = getarrayitem(self, i, array_items_ptr(self)); + PyObject *v = getarrayitem(self, i, self->data); if (v == NULL) goto error; PyList_SET_ITEM(list, i, v); @@ -2920,7 +2898,7 @@ array_ass_subscr_lock_held(PyObject *op, PyObject* item, PyObject* value) slicelength = 1; } else - return setarrayitem(self, i, value, array_items_ptr(self)); + return setarrayitem(self, i, value, self->data); } else if (PySlice_Check(item)) { if (PySlice_Unpack(item, &start, &stop, &step) < 0) { @@ -3069,7 +3047,7 @@ array_ass_subscr(PyObject *op, PyObject* item, PyObject* value) return -1; if (i < 0) i += PyArray_GET_SIZE(op); - return setarrayitem_maybe_locked(op, i, value); + return setarrayitem_threadsafe(op, i, value); } } @@ -3202,7 +3180,7 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) Py_DECREF(a); return NULL; } - if (setarrayitem(ap, i, v, ap->data->items) != 0) { + if (setarrayitem(ap, i, v, ap->data) != 0) { Py_DECREF(v); Py_DECREF(a); return NULL; @@ -3231,14 +3209,16 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) if (n > 0) { arrayobject *self = (arrayobject *)a; - self->data = arraydata_set_items(self->data, - (char *)ustr, n, sizeof(wchar_t), false); - PyMem_Free(ustr); + assert(self->data == NULL); + self->data = arraydata_alloc(n, sizeof(wchar_t)); if (self->data == NULL) { + PyMem_Free(ustr); Py_DECREF(a); PyErr_NoMemory(); return NULL; } + memcpy(self->data->items, (char *)ustr, n * sizeof(wchar_t)); + PyMem_Free(ustr); Py_SET_SIZE(self, n); } } @@ -3251,14 +3231,16 @@ array_new_internal_lock_held(PyTypeObject *type, PyObject *initial, int c) } arrayobject *self = (arrayobject *)a; - self->data = arraydata_set_items(self->data, - (char *)ustr, n, sizeof(Py_UCS4), false); - PyMem_Free(ustr); + assert(self->data == NULL); + self->data = arraydata_alloc(n, sizeof(Py_UCS4)); if (self->data == NULL) { + PyMem_Free(ustr); Py_DECREF(a); PyErr_NoMemory(); return NULL; } + memcpy(self->data->items, (char *)ustr, n * sizeof(Py_UCS4)); + PyMem_Free(ustr); Py_SET_SIZE(self, n); } } @@ -3508,7 +3490,7 @@ arrayiter_next(PyObject *op) assert(PyObject_TypeCheck(it, state->ArrayIterType)); assert(array_Check(ao, state)); #endif - PyObject *ret = getarrayitem_maybe_locked((PyObject *)ao, index); + PyObject *ret = getarrayitem_threadsafe((PyObject *)ao, index); if (ret != NULL) { FT_ATOMIC_STORE_SSIZE_RELAXED(it->index, index + 1); } From b5d219ebe2d8305701b5a0be379411c0e9b38628 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 17 Mar 2025 18:30:52 -0400 Subject: [PATCH 23/32] __declspec(align(8)) for Windows --- Modules/arraymodule.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index f33f6d4fea186c..85d50f3ab8cea1 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -42,11 +42,22 @@ struct arraydescr { int is_signed; }; +#ifdef _MSC_VER + +typedef __declspec(align(8)) struct { + Py_ssize_t allocated; + char items[]; +} arraydata; + +#else + typedef struct { Py_ssize_t allocated; _Alignas(8) char items[]; } arraydata; +#endif + typedef struct arrayobject { PyObject_VAR_HEAD arraydata *data; From 2c82071a7106d3b9ca9e6695edbc7a1c35db3694 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 17 Mar 2025 19:00:31 -0400 Subject: [PATCH 24/32] shut up check-c-globals --- Modules/arraymodule.c | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 85d50f3ab8cea1..0b9419be27f993 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -42,22 +42,14 @@ struct arraydescr { int is_signed; }; -#ifdef _MSC_VER - -typedef __declspec(align(8)) struct { - Py_ssize_t allocated; - char items[]; -} arraydata; - -#else - typedef struct { - Py_ssize_t allocated; - _Alignas(8) char items[]; + union { + Py_ssize_t allocated; + double __alignment_padding; /* In case Py_ssize_t is 4 bytes */ + }; + char items[]; } arraydata; -#endif - typedef struct arrayobject { PyObject_VAR_HEAD arraydata *data; From 1ba50e97343f4f9765c643b64d49c8322371718c Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 20 Mar 2025 07:17:49 -0400 Subject: [PATCH 25/32] alignment changes --- Modules/arraymodule.c | 13 ++++++++----- Tools/c-analyzer/c_parser/parser/_regexes.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 0b9419be27f993..dd018db779035c 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -42,12 +42,15 @@ struct arraydescr { int is_signed; }; +#ifdef MS_WINDOWS +#define Py_ALIGN_AS(x) __declspec(align(x)) +#else +#define Py_ALIGN_AS(x) _Alignas(x) +#endif + typedef struct { - union { - Py_ssize_t allocated; - double __alignment_padding; /* In case Py_ssize_t is 4 bytes */ - }; - char items[]; + Py_ssize_t allocated; + Py_ALIGN_AS(8) char items[]; } arraydata; typedef struct arrayobject { diff --git a/Tools/c-analyzer/c_parser/parser/_regexes.py b/Tools/c-analyzer/c_parser/parser/_regexes.py index c1a8ab3ad2f15d..76f0f71a2f7724 100644 --- a/Tools/c-analyzer/c_parser/parser/_regexes.py +++ b/Tools/c-analyzer/c_parser/parser/_regexes.py @@ -59,6 +59,8 @@ def _ind(text, level=1, edges='both'): register | static | _Thread_local | + _Alignas | + _Alignof | typedef | const | @@ -154,6 +156,17 @@ def _ind(text, level=1, edges='both'): TYPE_QUALIFIER = r'(?: \b (?: const | volatile ) \b )' PTR_QUALIFIER = rf'(?: [*] (?: \s* {TYPE_QUALIFIER} )? )' +ALIGNMENT_SPECIFIER = textwrap.dedent(r''' + # alignment specifier + (?: + _Alignas + \s* [(] + [^)]* + [)] + ) + # end alignment specifier + ''') + TYPE_SPEC = textwrap.dedent(rf''' # type spec (?: @@ -318,6 +331,9 @@ def _ind(text, level=1, edges='both'): (?: # typed member (?: + (?: # + \s* {ALIGNMENT_SPECIFIER} \s* + )? # Technically it doesn't have to have a type... (?: # (?: {TYPE_QUALIFIER} \s* )? From 576aebf4ba5ed9dbae32c66d7df3305f1df950e8 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 20 Mar 2025 13:19:42 -0400 Subject: [PATCH 26/32] MS_WINDOWS -> _MSC_VER --- Modules/arraymodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index dd018db779035c..7155bebafb0c9f 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -42,7 +42,7 @@ struct arraydescr { int is_signed; }; -#ifdef MS_WINDOWS +#ifdef _MSC_VER #define Py_ALIGN_AS(x) __declspec(align(x)) #else #define Py_ALIGN_AS(x) _Alignas(x) From d4e53139d7b931bb171d881bf7b529d8521ea069 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Thu, 20 Mar 2025 13:38:16 -0400 Subject: [PATCH 27/32] #include "pycore_gc.h", something moved --- Modules/arraymodule.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 7155bebafb0c9f..5ade73348424fd 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -11,6 +11,7 @@ #include "pycore_bytesobject.h" // _PyBytes_Repeat #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_gc.h" // _PyObject_GC_IS_SHARED() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyatomic_ft_wrappers.h" From affae8e57009b25d7746a8b2cf0ae3bf8b4ec7db Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Tue, 25 Mar 2025 10:54:04 -0400 Subject: [PATCH 28/32] remove NULL check in arraydata_free() --- Modules/arraymodule.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 5ade73348424fd..0424c48e281230 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -172,9 +172,7 @@ arraydata_free(arraydata *data, bool use_qsbr) { #ifdef Py_GIL_DISABLED if (use_qsbr) { - if (data != NULL) { - _PyMem_FreeDelayed(data); - } + _PyMem_FreeDelayed(data); } else { PyMem_Free(data); From 2e7132e8fc0ab579c6d86fa77bbe818aa51dbb52 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 5 May 2025 07:15:04 -0400 Subject: [PATCH 29/32] change to use new _Py_ALIGN_AS() macro --- Include/pymacro.h | 50 +++++++++++++++++++++---------------------- Modules/arraymodule.c | 9 ++------ 2 files changed, 26 insertions(+), 33 deletions(-) diff --git a/Include/pymacro.h b/Include/pymacro.h index 218987a80b0d91..f0ca334e87545c 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -25,8 +25,8 @@ // _Py_ALIGN_AS: this compiler's spelling of `alignas` keyword, -// We currently use alignas for free-threaded builds only; additional compat -// checking would be great before we add it to the default build. +// additional compat checking would be great since we added it to the default +// build. // Standards/compiler support: // - `alignas` is a keyword in C23 and C++11. // - `_Alignas` is a keyword in C11 @@ -38,30 +38,28 @@ // unsupported platforms, we don't redefine _Py_ALIGN_AS if it's already // defined. Note that defining it wrong (including defining it to nothing) will // cause ABI incompatibilities. -#ifdef Py_GIL_DISABLED -# ifndef _Py_ALIGN_AS -# ifdef __cplusplus -# if __cplusplus >= 201103L -# define _Py_ALIGN_AS(V) alignas(V) -# elif defined(__GNUC__) || defined(__clang__) -# define _Py_ALIGN_AS(V) __attribute__((aligned(V))) -# elif defined(_MSC_VER) -# define _Py_ALIGN_AS(V) __declspec(align(V)) -# else -# define _Py_ALIGN_AS(V) alignas(V) -# endif -# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L -# define _Py_ALIGN_AS(V) alignas(V) -# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L -# define _Py_ALIGN_AS(V) _Alignas(V) -# elif (defined(__GNUC__) || defined(__clang__)) -# define _Py_ALIGN_AS(V) __attribute__((aligned(V))) -# elif defined(_MSC_VER) -# define _Py_ALIGN_AS(V) __declspec(align(V)) -# else -# define _Py_ALIGN_AS(V) _Alignas(V) -# endif -# endif +#ifndef _Py_ALIGN_AS +# ifdef __cplusplus +# if __cplusplus >= 201103L +# define _Py_ALIGN_AS(V) alignas(V) +# elif defined(__GNUC__) || defined(__clang__) +# define _Py_ALIGN_AS(V) __attribute__((aligned(V))) +# elif defined(_MSC_VER) +# define _Py_ALIGN_AS(V) __declspec(align(V)) +# else +# define _Py_ALIGN_AS(V) alignas(V) +# endif +# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +# define _Py_ALIGN_AS(V) alignas(V) +# elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +# define _Py_ALIGN_AS(V) _Alignas(V) +# elif (defined(__GNUC__) || defined(__clang__)) +# define _Py_ALIGN_AS(V) __attribute__((aligned(V))) +# elif defined(_MSC_VER) +# define _Py_ALIGN_AS(V) __declspec(align(V)) +# else +# define _Py_ALIGN_AS(V) _Alignas(V) +# endif #endif /* Minimum value between x and y */ diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 3d9387855d293f..4d1c9ab28a5fe7 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -43,15 +43,10 @@ struct arraydescr { int is_signed; }; -#ifdef _MSC_VER -#define Py_ALIGN_AS(x) __declspec(align(x)) -#else -#define Py_ALIGN_AS(x) _Alignas(x) -#endif - typedef struct { Py_ssize_t allocated; - Py_ALIGN_AS(8) char items[]; + _Py_ALIGN_AS(8) + char items[]; } arraydata; typedef struct arrayobject { From 1a509813d640d9c610facb0172479899c931bc54 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Wed, 11 Jun 2025 09:32:04 -0400 Subject: [PATCH 30/32] requested changes --- Modules/arraymodule.c | 81 ++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 51 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 7687302458a6ab..a8cbb731e20fba 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -153,10 +153,14 @@ enum machine_format_code { static arraydata * arraydata_alloc(Py_ssize_t size, int itemsize) { - arraydata *data = (arraydata *)PyMem_Malloc(sizeof(arraydata) + size * itemsize); + size_t bufsize = sizeof(arraydata) + size * itemsize; + arraydata *data = (arraydata *)PyMem_Malloc(bufsize); if (data == NULL) { return NULL; } +#ifdef Py_DEBUG + memset(data, 0, bufsize); +#endif data->allocated = size; return data; } @@ -191,7 +195,7 @@ arraydata_realloc(arraydata *data, Py_ssize_t size, int itemsize) #endif -static char * +static inline char * array_items_ptr(arrayobject *self) { return self->data == NULL ? NULL : self->data->items; @@ -316,9 +320,8 @@ b_setitem(char *items, Py_ssize_t i, PyObject *v) /* PyArg_Parse's 'b' formatter is for an unsigned char, therefore must use the next size up that is signed ('h') and manually do the overflow checking */ - if (!PyArg_Parse(v, "h;array item must be integer", &x)) { + if (!PyArg_Parse(v, "h;array item must be integer", &x)) return -1; - } else if (x < -128) { PyErr_SetString(PyExc_OverflowError, "signed char is less than minimum"); @@ -329,9 +332,8 @@ b_setitem(char *items, Py_ssize_t i, PyObject *v) "signed char is greater than maximum"); return -1; } - if (i >= 0) { + if (i >= 0) ((char *)items)[i] = (char)x; - } return 0; } @@ -347,12 +349,10 @@ BB_setitem(char *items, Py_ssize_t i, PyObject *v) { unsigned char x; /* 'B' == unsigned char, maps to PyArg_Parse's 'b' formatter */ - if (!PyArg_Parse(v, "b;array item must be integer", &x)) { + if (!PyArg_Parse(v, "b;array item must be integer", &x)) return -1; - } - if (i >= 0) { + if (i >= 0) ((unsigned char *)items)[i] = x; - } return 0; } @@ -441,12 +441,10 @@ h_setitem(char *items, Py_ssize_t i, PyObject *v) { short x; /* 'h' == signed short, maps to PyArg_Parse's 'h' formatter */ - if (!PyArg_Parse(v, "h;array item must be integer", &x)) { + if (!PyArg_Parse(v, "h;array item must be integer", &x)) return -1; - } - if (i >= 0) { + if (i >= 0) ((short *)items)[i] = x; - } return 0; } @@ -462,9 +460,8 @@ HH_setitem(char *items, Py_ssize_t i, PyObject *v) int x; /* PyArg_Parse's 'h' formatter is for a signed short, therefore must use the next size up and manually do the overflow checking */ - if (!PyArg_Parse(v, "i;array item must be integer", &x)) { + if (!PyArg_Parse(v, "i;array item must be integer", &x)) return -1; - } else if (x < 0) { PyErr_SetString(PyExc_OverflowError, "unsigned short is less than minimum"); @@ -475,9 +472,8 @@ HH_setitem(char *items, Py_ssize_t i, PyObject *v) "unsigned short is greater than maximum"); return -1; } - if (i >= 0) { + if (i >= 0) ((short *)items)[i] = (short)x; - } return 0; } @@ -492,12 +488,10 @@ i_setitem(char *items, Py_ssize_t i, PyObject *v) { int x; /* 'i' == signed int, maps to PyArg_Parse's 'i' formatter */ - if (!PyArg_Parse(v, "i;array item must be integer", &x)) { + if (!PyArg_Parse(v, "i;array item must be integer", &x)) return -1; - } - if (i >= 0) { + if (i >= 0) ((int *)items)[i] = x; - } return 0; } @@ -536,9 +530,8 @@ II_setitem(char *items, Py_ssize_t i, PyObject *v) } return -1; } - if (i >= 0) { + if (i >= 0) ((unsigned int *)items)[i] = (unsigned int)x; - } if (do_decref) { Py_DECREF(v); @@ -556,12 +549,10 @@ static int l_setitem(char *items, Py_ssize_t i, PyObject *v) { long x; - if (!PyArg_Parse(v, "l;array item must be integer", &x)) { + if (!PyArg_Parse(v, "l;array item must be integer", &x)) return -1; - } - if (i >= 0) { + if (i >= 0) ((long *)items)[i] = x; - } return 0; } @@ -591,9 +582,8 @@ LL_setitem(char *items, Py_ssize_t i, PyObject *v) } return -1; } - if (i >= 0) { + if (i >= 0) ((unsigned long *)items)[i] = x; - } if (do_decref) { Py_DECREF(v); @@ -611,12 +601,10 @@ static int q_setitem(char *items, Py_ssize_t i, PyObject *v) { long long x; - if (!PyArg_Parse(v, "L;array item must be integer", &x)) { + if (!PyArg_Parse(v, "L;array item must be integer", &x)) return -1; - } - if (i >= 0) { + if (i >= 0) ((long long *)items)[i] = x; - } return 0; } @@ -647,9 +635,8 @@ QQ_setitem(char *items, Py_ssize_t i, PyObject *v) } return -1; } - if (i >= 0) { + if (i >= 0) ((unsigned long long *)items)[i] = x; - } if (do_decref) { Py_DECREF(v); @@ -667,12 +654,10 @@ static int f_setitem(char *items, Py_ssize_t i, PyObject *v) { float x; - if (!PyArg_Parse(v, "f;array item must be float", &x)) { + if (!PyArg_Parse(v, "f;array item must be float", &x)) return -1; - } - if (i >= 0) { + if (i >= 0) ((float *)items)[i] = x; - } return 0; } @@ -686,12 +671,10 @@ static int d_setitem(char *items, Py_ssize_t i, PyObject *v) { double x; - if (!PyArg_Parse(v, "d;array item must be float", &x)) { + if (!PyArg_Parse(v, "d;array item must be float", &x)) return -1; - } - if (i >= 0) { + if (i >= 0) ((double *)items)[i] = x; - } return 0; } @@ -700,11 +683,9 @@ d_setitem(char *items, Py_ssize_t i, PyObject *v) code##_compareitems(const void *lhs, const void *rhs, Py_ssize_t length) \ { \ const type *a = lhs, *b = rhs; \ - for (Py_ssize_t i = 0; i < length; ++i) { \ - if (a[i] != b[i]) { \ + for (Py_ssize_t i = 0; i < length; ++i) \ + if (a[i] != b[i]) \ return a[i] < b[i] ? -1 : 1; \ - } \ - } \ return 0; \ } @@ -984,9 +965,7 @@ array_dealloc(PyObject *op) arrayobject *self = arrayobject_CAST(op); if (self->ob_exports > 0) { - PyErr_SetString(PyExc_SystemError, - "deallocated array object has exported buffers"); - PyErr_WriteUnraisable(NULL); + PyErr_FormatUnraisable("deallocated array object has exported buffers"); } if (self->weakreflist != NULL) { PyObject_ClearWeakRefs(op); From f023e7718e9cf242c4e13c1ead0328c2cd81bf7c Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Fri, 13 Jun 2025 12:41:09 -0400 Subject: [PATCH 31/32] shut up UBSan --- Modules/arraymodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index a8cbb731e20fba..fcb3b24ab3319a 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -857,7 +857,7 @@ setarrayitem(arrayobject *ap, Py_ssize_t i, PyObject *v, arraydata *data) #endif } #endif - return (*ap->ob_descr->setitem)(data->items, i, v); + return (*ap->ob_descr->setitem)(data == NULL ? NULL : data->items, i, v); } static int From 6d50e9a64f76c6209e4676d8dbfd43a23196696e Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 6 Jul 2025 07:58:23 -0400 Subject: [PATCH 32/32] add size to _PyMem_FreeDelayed() --- Modules/arraymodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index b0a768b7db5afe..80f2d6aebaf04e 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -171,7 +171,7 @@ arraydata_free(arraydata *data, bool use_qsbr) { #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeDelayed(data); + _PyMem_FreeDelayed(data, data == NULL ? 0 : data->allocated); } else { PyMem_Free(data);