From bdaef6b92c0be66cb60f22566fcba1b54237d264 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 7 Mar 2024 16:47:42 -0700 Subject: [PATCH 01/19] Fix a comment. --- Lib/test/test_interpreters/test_queues.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index d16d294b82d044..8ab9ebb354712a 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -28,9 +28,9 @@ def tearDown(self): class LowLevelTests(TestBase): - # The behaviors in the low-level module is important in as much - # as it is exercised by the high-level module. Therefore the - # most # important testing happens in the high-level tests. + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. # These low-level tests cover corner cases that are not # encountered by the high-level module, thus they # mostly shouldn't matter as much. From 6342635375bfa24bb30e1bfae80fead50a5916b9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 21 Mar 2024 14:50:00 -0600 Subject: [PATCH 02/19] Add PyInterpreterConfig helpers. --- Include/internal/pycore_pylifecycle.h | 13 ++ Python/initconfig.c | 244 +++++++++++++++++++++++++- Python/pystate.c | 22 +++ 3 files changed, 275 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index c675098685764c..5f24324830f06f 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -116,6 +116,19 @@ PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category); // Export for special main.c string compiling with source tracebacks int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags); + +/* interpreter config */ + +extern int _PyInterpreterState_ResolveConfig( + PyInterpreterState *, + PyInterpreterConfig *); +extern PyObject * _PyInterpreterConfig_AsDict(PyInterpreterConfig *); +extern int _PyInterpreterConfig_FromDict(PyObject *, PyInterpreterConfig *); +extern int _PyInterpreterConfig_UpdateFromDict( + PyObject *, + PyInterpreterConfig *); + + #ifdef __cplusplus } #endif diff --git a/Python/initconfig.c b/Python/initconfig.c index 215d6a1d4e0dba..9f2e42dfd87b04 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1098,15 +1098,30 @@ _PyConfig_AsDict(const PyConfig *config) } -static PyObject* -config_dict_get(PyObject *dict, const char *name) +static inline int +_config_dict_get(PyObject *dict, const char *name, PyObject **p_item) { PyObject *item; if (PyDict_GetItemStringRef(dict, name, &item) < 0) { - return NULL; + return -1; } if (item == NULL) { - PyErr_Format(PyExc_ValueError, "missing config key: %s", name); + // We do not set an exception. + return -1; + } + *p_item = item; + return 0; +} + + +static PyObject* +config_dict_get(PyObject *dict, const char *name) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, "missing config key: %s", name); + } return NULL; } return item; @@ -3249,3 +3264,224 @@ _Py_DumpPathConfig(PyThreadState *tstate) _PyErr_SetRaisedException(tstate, exc); } + + +/* PyInterpreterConfig + + This isn't exactly the right place for API related to PyInterpreterConfig, + but neither pylifecycle.c nor pystate.c are quite right either. + For now, this is as good a place as any. +*/ + +static const char * +gil_flag_to_str(int flag) +{ + switch (flag) { + case PyInterpreterConfig_DEFAULT_GIL: + return "default"; + case PyInterpreterConfig_SHARED_GIL: + return "shared"; + case PyInterpreterConfig_OWN_GIL: + return "own"; + default: + PyErr_SetString(PyExc_SystemError, + "invalid interpreter config 'gil' value"); + return NULL; + } +} + +static int +gil_flag_from_str(const char *str, int *p_flag) +{ + int flag; + if (str == NULL || strcmp(str, "") == 0) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "default") == 0) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "shared") == 0) { + flag = PyInterpreterConfig_SHARED_GIL; + } + else if (strcmp(str, "own") == 0) { + flag = PyInterpreterConfig_OWN_GIL; + } + else { + PyErr_SetString(PyExc_SystemError, + "invalid interpreter config 'gil' value"); + return -1; + } + *p_flag = flag; + return 0; +} + +PyObject * +_PyInterpreterConfig_AsDict(PyInterpreterConfig *config) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + +#define ADD(NAME, OBJ) \ + do { \ + int res = PyDict_SetItemString(dict, NAME, (OBJ)); \ + Py_DECREF(OBJ); \ + if (res < 0) { \ + goto error; \ + } \ + } while (0) +#define ADD_BOOL(FIELD) \ + ADD(#FIELD, Py_NewRef(config->FIELD ? Py_True : Py_False)) +#define ADD_STR(FIELD, STR) \ + do { \ + if (STR == NULL) { \ + goto error; \ + } \ + PyObject *obj = PyUnicode_FromString(#FIELD); \ + if (obj == NULL) { \ + goto error; \ + } \ + ADD(#FIELD, obj); \ + } while (0) + + ADD_BOOL(use_main_obmalloc); + ADD_BOOL(allow_fork); + ADD_BOOL(allow_exec); + ADD_BOOL(allow_threads); + ADD_BOOL(allow_daemon_threads); + ADD_BOOL(check_multi_interp_extensions); + + ADD_STR(gil, gil_flag_to_str(config->gil)); + +#undef ADD_STR +#undef ADD_BOOL +#undef ADD + + return dict; + +error: + Py_DECREF(dict); + return NULL; +} + +static int +_config_dict_get_bool(PyObject *dict, const char *name, int *p_flag) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + int flag = PyObject_IsTrue(item); + Py_DECREF(item); + if (flag < 0) { + return -1; + } + *p_flag = flag; + return 0; +} + +static int +_config_dict_copy_str(PyObject *dict, const char *name, + char *buf, size_t bufsize) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + if (!PyUnicode_Check(item)) { + Py_DECREF(item); + config_dict_invalid_type(name); + return -1; + } + strncpy(buf, PyUnicode_AsUTF8(item), bufsize-1); + buf[bufsize-1] = '\0'; + Py_DECREF(item); + return 0; +} + +static int +interp_config_from_dict(PyObject *dict, PyInterpreterConfig *config, + bool missing_allowed) +{ + Py_ssize_t unused = PyDict_GET_SIZE(dict); + +#define CHECK(NAME) \ + do { \ + if (!PyErr_Occurred()) { \ + if (!missing_allowed) { \ + (void)config_dict_get(dict, NAME); \ + assert(PyErr_Occurred()); \ + return -1; \ + } \ + } \ + } while (0) +#define COPY_BOOL(FIELD) \ + do { \ + int flag; \ + if (_config_dict_get_bool(dict, #FIELD, &flag) < 0) { \ + CHECK(#FIELD); \ + } \ + else { \ + config->FIELD = flag; \ + unused -= 1; \ + } \ + } while (0) + + COPY_BOOL(use_main_obmalloc); + COPY_BOOL(allow_fork); + COPY_BOOL(allow_exec); + COPY_BOOL(allow_threads); + COPY_BOOL(allow_daemon_threads); + COPY_BOOL(check_multi_interp_extensions); + + // PyInterpreterConfig.gil + char buf[20]; + if (_config_dict_copy_str(dict, "gil", buf, 20) < 0) { + CHECK("gil"); + } + else { + int flag; + if (gil_flag_from_str(buf, &flag) < 0) { + return -1; + } + config->gil = flag; + unused -= 1; + } + +#undef COPY_BOOL +#undef CHECK + + if (unused > 0) { + PyErr_Format(PyExc_ValueError, + "dict as %d extra items", unused); + return -1; + } + return 0; +} + +int +_PyInterpreterConfig_FromDict(PyObject *dict, PyInterpreterConfig *config) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, false) < 0) { + return -1; + } + return 0; +} + +int +_PyInterpreterConfig_UpdateFromDict(PyObject *dict, PyInterpreterConfig *config) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, true) < 0) { + return -1; + } + return 0; +} diff --git a/Python/pystate.c b/Python/pystate.c index 47d327ae28933b..4a2106f19701c8 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1122,6 +1122,28 @@ PyInterpreterState_GetDict(PyInterpreterState *interp) return interp->dict; } +int +_PyInterpreterState_ResolveConfig(PyInterpreterState *interp, + PyInterpreterConfig *config) +{ + // Populate the config by re-constructing the values from the interpreter. + *config = (PyInterpreterConfig){ +#define FLAG(flag) \ + (interp->feature_flags & Py_RTFLAGS_ ## flag) + .use_main_obmalloc = FLAG(USE_MAIN_OBMALLOC), + .allow_fork = FLAG(FORK), + .allow_exec = FLAG(EXEC), + .allow_threads = FLAG(THREADS), + .allow_daemon_threads = FLAG(DAEMON_THREADS), + .check_multi_interp_extensions = FLAG(MULTI_INTERP_EXTENSIONS), +#undef FLAG + .gil = interp->ceval.own_gil + ? PyInterpreterConfig_OWN_GIL + : PyInterpreterConfig_SHARED_GIL, + }; + return 0; +} + //---------- // interp ID From c48dd00f4ceb53e3c86da3782b90b5e8c160ce60 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 21 Mar 2024 17:48:22 -0600 Subject: [PATCH 03/19] Add config helpers to _testinternalcapi. --- Include/internal/pycore_pylifecycle.h | 11 ++-- Modules/_testinternalcapi.c | 91 +++++++++++++++++++++++++++ Python/initconfig.c | 2 +- 3 files changed, 99 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 5f24324830f06f..a230cd2c5b927a 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -119,14 +119,17 @@ int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCo /* interpreter config */ -extern int _PyInterpreterState_ResolveConfig( +// Export for _testinternalcapi shared extension +PyAPI_FUNC(int) _PyInterpreterState_ResolveConfig( PyInterpreterState *, PyInterpreterConfig *); -extern PyObject * _PyInterpreterConfig_AsDict(PyInterpreterConfig *); -extern int _PyInterpreterConfig_FromDict(PyObject *, PyInterpreterConfig *); -extern int _PyInterpreterConfig_UpdateFromDict( +PyAPI_FUNC(PyObject *) _PyInterpreterConfig_AsDict(PyInterpreterConfig *); +PyAPI_FUNC(int) _PyInterpreterConfig_FromDict( PyObject *, PyInterpreterConfig *); +PyAPI_FUNC(int) _PyInterpreterConfig_UpdateFromDict( + PyInterpreterConfig *, + PyObject *); #ifdef __cplusplus diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index e1717f7a66b1de..1e3cee21642408 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -23,10 +23,12 @@ #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() +#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pylifecycle.h" // _PyInterpreterState_ResolveConfig() #include "pycore_pystate.h" // _PyThreadState_GET() #include "clinic/_testinternalcapi.c.h" @@ -829,6 +831,7 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, } +// Maybe this could be replaced by get_interpreter_config()? static PyObject * get_interp_settings(PyObject *self, PyObject *args) { @@ -1376,6 +1379,91 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } +static PyObject * +get_interpreter_config(PyObject *self, PyObject *args) +{ + PyObject *idobj = NULL; + if (!PyArg_ParseTuple(args, "|O:get_interpreter_config", &idobj)) { + return NULL; + } + + PyInterpreterState *interp; + if (idobj == NULL) { + interp = PyInterpreterState_Get(); + } + else { + interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + } + + PyInterpreterConfig config; + if (_PyInterpreterState_ResolveConfig(interp, &config) < 0) { + return NULL; + } + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +static PyObject * +new_interpreter_config(PyObject *self, PyObject *args, PyObject *kwargs) +{ + const char *initialized = NULL; + PyObject *overrides = NULL; + static char *kwlist[] = {"initialized", "overrides", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "|sO:new_interpreter_config", kwlist, + &initialized, &overrides)) + { + return NULL; + } + if (initialized == NULL + || strcmp(initialized, "") == 0 + || strcmp(initialized, "default") == 0) + { + initialized = "isolated"; + } + + PyInterpreterConfig config; + if (strcmp(initialized, "isolated") == 0) { + config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + } + else if (strcmp(initialized, "legacy") == 0) { + config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + } + else if (strcmp(initialized, "empty") == 0) { + config = (PyInterpreterConfig){0}; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported initialized arg '%s'", initialized); + return NULL; + } + + if (overrides != NULL) { + if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { + return NULL; + } + } + + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + + /* To run some code in a sub-interpreter. */ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) @@ -1858,6 +1946,9 @@ static PyMethodDef module_functions[] = { {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, + {"get_interpreter_config", get_interpreter_config, METH_VARARGS}, + {"new_interpreter_config", _PyCFunction_CAST(new_interpreter_config), + METH_VARARGS | METH_KEYWORDS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, diff --git a/Python/initconfig.c b/Python/initconfig.c index 9f2e42dfd87b04..3ef47e9c101423 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -3474,7 +3474,7 @@ _PyInterpreterConfig_FromDict(PyObject *dict, PyInterpreterConfig *config) } int -_PyInterpreterConfig_UpdateFromDict(PyObject *dict, PyInterpreterConfig *config) +_PyInterpreterConfig_UpdateFromDict(PyInterpreterConfig *config, PyObject *dict) { if (!PyDict_Check(dict)) { PyErr_SetString(PyExc_TypeError, "dict expected"); From 0796fe99ad1a4f140f391dba57fa7721a5362553 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 21 Mar 2024 19:00:56 -0600 Subject: [PATCH 04/19] Use the new helpers in run_in_subinterp_with_config(). --- Lib/test/support/__init__.py | 15 +++++- Lib/test/test_import/__init__.py | 12 ++++- Modules/_testinternalcapi.c | 80 +++++++------------------------- Python/initconfig.c | 39 ++++++++++++---- 4 files changed, 70 insertions(+), 76 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index a1c7987fa0db47..e99555efca2622 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1728,8 +1728,19 @@ def run_in_subinterp_with_config(code, *, own_gil=None, **config): import _testinternalcapi if own_gil is not None: assert 'gil' not in config, (own_gil, config) - config['gil'] = 2 if own_gil else 1 - return _testinternalcapi.run_in_subinterp_with_config(code, **config) + config['gil'] = 'own' if own_gil else 'shared' + else: + gil = config['gil'] + if gil == 0: + config['gil'] = 'default' + elif gil == 1: + config['gil'] = 'shared' + elif gil == 2: + config['gil'] = 'own' + else: + raise NotImplementedError(gil) + config = types.SimpleNamespace(**config) + return _testinternalcapi.run_in_subinterp_with_config(code, config) def _check_tracemalloc(): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 4deed7f3ba2522..81ec700d9755ce 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1823,15 +1823,19 @@ def check_compatible_fresh(self, name, *, strict=False, isolated=False): **(self.ISOLATED if isolated else self.NOT_ISOLATED), check_multi_interp_extensions=strict, ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' import _testinternalcapi, sys assert ( {name!r} in sys.builtin_module_names or {name!r} not in sys.modules ), repr({name!r}) + config = type(sys.implementation)(**{kwargs}) ret = _testinternalcapi.run_in_subinterp_with_config( {self.import_script(name, "sys.stdout.fileno()")!r}, - **{kwargs}, + config, ) assert ret == 0, ret ''')) @@ -1847,12 +1851,16 @@ def check_incompatible_fresh(self, name, *, isolated=False): **(self.ISOLATED if isolated else self.NOT_ISOLATED), check_multi_interp_extensions=True, ) + gil = kwargs['gil'] + kwargs['gil'] = 'default' if gil == 0 else ( + 'shared' if gil == 1 else 'own' if gil == 2 else gil) _, out, err = script_helper.assert_python_ok('-c', textwrap.dedent(f''' import _testinternalcapi, sys assert {name!r} not in sys.modules, {name!r} + config = type(sys.implementation)(**{kwargs}) ret = _testinternalcapi.run_in_subinterp_with_config( {self.import_script(name, "sys.stdout.fileno()")!r}, - **{kwargs}, + config, ) assert ret == 0, ret ''')) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 1e3cee21642408..aff6af9e47c75d 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1469,78 +1469,32 @@ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) { const char *code; - int use_main_obmalloc = -1; - int allow_fork = -1; - int allow_exec = -1; - int allow_threads = -1; - int allow_daemon_threads = -1; - int check_multi_interp_extensions = -1; - int gil = -1; - int r; - PyThreadState *substate, *mainstate; - /* only initialise 'cflags.cf_flags' to test backwards compatibility */ - PyCompilerFlags cflags = {0}; - - static char *kwlist[] = {"code", - "use_main_obmalloc", - "allow_fork", - "allow_exec", - "allow_threads", - "allow_daemon_threads", - "check_multi_interp_extensions", - "gil", - NULL}; + PyObject *configobj; + static char *kwlist[] = {"code", "config", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "s$ppppppi:run_in_subinterp_with_config", kwlist, - &code, &use_main_obmalloc, - &allow_fork, &allow_exec, - &allow_threads, &allow_daemon_threads, - &check_multi_interp_extensions, - &gil)) { - return NULL; - } - if (use_main_obmalloc < 0) { - PyErr_SetString(PyExc_ValueError, "missing use_main_obmalloc"); - return NULL; - } - if (allow_fork < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_fork"); - return NULL; - } - if (allow_exec < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_exec"); - return NULL; - } - if (allow_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_threads"); - return NULL; - } - if (gil < 0) { - PyErr_SetString(PyExc_ValueError, "missing gil"); + "sO:run_in_subinterp_with_config", kwlist, + &code, &configobj)) + { return NULL; } - if (allow_daemon_threads < 0) { - PyErr_SetString(PyExc_ValueError, "missing allow_daemon_threads"); + + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); return NULL; } - if (check_multi_interp_extensions < 0) { - PyErr_SetString(PyExc_ValueError, "missing check_multi_interp_extensions"); + PyInterpreterConfig config = {0}; + int res = _PyInterpreterConfig_FromDict(dict, &config); + Py_DECREF(dict); + if (res < 0) { return NULL; } - mainstate = PyThreadState_Get(); + PyThreadState *mainstate = PyThreadState_Get(); PyThreadState_Swap(NULL); - const PyInterpreterConfig config = { - .use_main_obmalloc = use_main_obmalloc, - .allow_fork = allow_fork, - .allow_exec = allow_exec, - .allow_threads = allow_threads, - .allow_daemon_threads = allow_daemon_threads, - .check_multi_interp_extensions = check_multi_interp_extensions, - .gil = gil, - }; + PyThreadState *substate; PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); if (PyStatus_Exception(status)) { /* Since no new thread state was created, there is no exception to @@ -1554,7 +1508,9 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } assert(substate != NULL); - r = PyRun_SimpleStringFlags(code, &cflags); + /* only initialise 'cflags.cf_flags' to test backwards compatibility */ + PyCompilerFlags cflags = {0}; + int r = PyRun_SimpleStringFlags(code, &cflags); Py_EndInterpreter(substate); PyThreadState_Swap(mainstate); diff --git a/Python/initconfig.c b/Python/initconfig.c index 3ef47e9c101423..da5e523f0dfe83 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -3401,18 +3401,27 @@ _config_dict_copy_str(PyObject *dict, const char *name, } static int -interp_config_from_dict(PyObject *dict, PyInterpreterConfig *config, +interp_config_from_dict(PyObject *origdict, PyInterpreterConfig *config, bool missing_allowed) { - Py_ssize_t unused = PyDict_GET_SIZE(dict); + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return -1; + } + if (PyDict_Update(dict, origdict) < 0) { + goto error; + } #define CHECK(NAME) \ do { \ - if (!PyErr_Occurred()) { \ + if (PyErr_Occurred()) { \ + goto error; \ + } \ + else { \ if (!missing_allowed) { \ (void)config_dict_get(dict, NAME); \ assert(PyErr_Occurred()); \ - return -1; \ + goto error; \ } \ } \ } while (0) @@ -3424,7 +3433,7 @@ interp_config_from_dict(PyObject *dict, PyInterpreterConfig *config, } \ else { \ config->FIELD = flag; \ - unused -= 1; \ + (void)PyDict_PopString(dict, #FIELD, NULL); \ } \ } while (0) @@ -3443,21 +3452,31 @@ interp_config_from_dict(PyObject *dict, PyInterpreterConfig *config, else { int flag; if (gil_flag_from_str(buf, &flag) < 0) { - return -1; + goto error; } config->gil = flag; - unused -= 1; + (void)PyDict_PopString(dict, "gil", NULL); } #undef COPY_BOOL #undef CHECK - if (unused > 0) { + Py_ssize_t unused = PyDict_GET_SIZE(dict); + if (unused == 1) { PyErr_Format(PyExc_ValueError, - "dict as %d extra items", unused); - return -1; + "config dict has 1 extra item (%R)", dict); + goto error; + } + else if (unused > 0) { + PyErr_Format(PyExc_ValueError, + "config dict has %d extra items (%R)", unused, dict); + goto error; } return 0; + +error: + Py_DECREF(dict); + return -1; } int From 8993b41757dd2c3a250b3bb655d0ee28e55af22d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 11:17:13 -0600 Subject: [PATCH 05/19] Move the PyInterpreterConfig utils to their own file. --- Makefile.pre.in | 5 + PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Python/config_common.h | 36 ++++ Python/initconfig.c | 280 +------------------------ Python/interpconfig.c | 241 +++++++++++++++++++++ 8 files changed, 293 insertions(+), 277 deletions(-) create mode 100644 Python/config_common.h create mode 100644 Python/interpconfig.c diff --git a/Makefile.pre.in b/Makefile.pre.in index c454f31aae1e57..1c34adce9a27f4 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -434,6 +434,7 @@ PYTHON_OBJS= \ Python/import.o \ Python/importdl.o \ Python/initconfig.o \ + Python/interpconfig.o \ Python/instrumentation.o \ Python/intrinsics.o \ Python/jit.o \ @@ -1680,6 +1681,10 @@ Modules/_xxinterpchannelsmodule.o: $(srcdir)/Modules/_xxinterpchannelsmodule.c $ Python/crossinterp.o: $(srcdir)/Python/crossinterp.c $(srcdir)/Python/crossinterp_data_lookup.h $(srcdir)/Python/crossinterp_exceptions.h +Python/initconfig.o: $(srcdir)/Python/initconfig.c $(srcdir)/Python/config_common.h + +Python/interpconfig.o: $(srcdir)/Python/interpconfig.c $(srcdir)/Python/config_common.h + Python/dynload_shlib.o: $(srcdir)/Python/dynload_shlib.c Makefile $(CC) -c $(PY_CORE_CFLAGS) \ -DSOABI='"$(SOABI)"' \ diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 82471e0f140ec3..9c82fcf021bb55 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -222,6 +222,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 97c52fdadf7c05..63b033a0350b20 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -229,6 +229,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index c944bbafdba7e5..97f779515a86d2 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -586,6 +586,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 0afad125ce1e97..8e1bd456cf5352 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1340,6 +1340,9 @@ Python + + Python + Source Files diff --git a/Python/config_common.h b/Python/config_common.h new file mode 100644 index 00000000000000..e749bd4bf0dc68 --- /dev/null +++ b/Python/config_common.h @@ -0,0 +1,36 @@ + +static inline int +_config_dict_get(PyObject *dict, const char *name, PyObject **p_item) +{ + PyObject *item; + if (PyDict_GetItemStringRef(dict, name, &item) < 0) { + return -1; + } + if (item == NULL) { + // We do not set an exception. + return -1; + } + *p_item = item; + return 0; +} + + +static PyObject* +config_dict_get(PyObject *dict, const char *name) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, "missing config key: %s", name); + } + return NULL; + } + return item; +} + + +static void +config_dict_invalid_type(const char *name) +{ + PyErr_Format(PyExc_TypeError, "invalid config type: %s", name); +} diff --git a/Python/initconfig.c b/Python/initconfig.c index da5e523f0dfe83..d91a8199b544dc 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -24,6 +24,9 @@ # endif #endif +#include "config_common.h" + + /* --- PyConfig spec ---------------------------------------------- */ typedef enum { @@ -1098,36 +1101,6 @@ _PyConfig_AsDict(const PyConfig *config) } -static inline int -_config_dict_get(PyObject *dict, const char *name, PyObject **p_item) -{ - PyObject *item; - if (PyDict_GetItemStringRef(dict, name, &item) < 0) { - return -1; - } - if (item == NULL) { - // We do not set an exception. - return -1; - } - *p_item = item; - return 0; -} - - -static PyObject* -config_dict_get(PyObject *dict, const char *name) -{ - PyObject *item; - if (_config_dict_get(dict, name, &item) < 0) { - if (!PyErr_Occurred()) { - PyErr_Format(PyExc_ValueError, "missing config key: %s", name); - } - return NULL; - } - return item; -} - - static void config_dict_invalid_value(const char *name) { @@ -1135,13 +1108,6 @@ config_dict_invalid_value(const char *name) } -static void -config_dict_invalid_type(const char *name) -{ - PyErr_Format(PyExc_TypeError, "invalid config type: %s", name); -} - - static int config_dict_get_int(PyObject *dict, const char *name, int *result) { @@ -3264,243 +3230,3 @@ _Py_DumpPathConfig(PyThreadState *tstate) _PyErr_SetRaisedException(tstate, exc); } - - -/* PyInterpreterConfig - - This isn't exactly the right place for API related to PyInterpreterConfig, - but neither pylifecycle.c nor pystate.c are quite right either. - For now, this is as good a place as any. -*/ - -static const char * -gil_flag_to_str(int flag) -{ - switch (flag) { - case PyInterpreterConfig_DEFAULT_GIL: - return "default"; - case PyInterpreterConfig_SHARED_GIL: - return "shared"; - case PyInterpreterConfig_OWN_GIL: - return "own"; - default: - PyErr_SetString(PyExc_SystemError, - "invalid interpreter config 'gil' value"); - return NULL; - } -} - -static int -gil_flag_from_str(const char *str, int *p_flag) -{ - int flag; - if (str == NULL || strcmp(str, "") == 0) { - flag = PyInterpreterConfig_DEFAULT_GIL; - } - else if (strcmp(str, "default") == 0) { - flag = PyInterpreterConfig_DEFAULT_GIL; - } - else if (strcmp(str, "shared") == 0) { - flag = PyInterpreterConfig_SHARED_GIL; - } - else if (strcmp(str, "own") == 0) { - flag = PyInterpreterConfig_OWN_GIL; - } - else { - PyErr_SetString(PyExc_SystemError, - "invalid interpreter config 'gil' value"); - return -1; - } - *p_flag = flag; - return 0; -} - -PyObject * -_PyInterpreterConfig_AsDict(PyInterpreterConfig *config) -{ - PyObject *dict = PyDict_New(); - if (dict == NULL) { - return NULL; - } - -#define ADD(NAME, OBJ) \ - do { \ - int res = PyDict_SetItemString(dict, NAME, (OBJ)); \ - Py_DECREF(OBJ); \ - if (res < 0) { \ - goto error; \ - } \ - } while (0) -#define ADD_BOOL(FIELD) \ - ADD(#FIELD, Py_NewRef(config->FIELD ? Py_True : Py_False)) -#define ADD_STR(FIELD, STR) \ - do { \ - if (STR == NULL) { \ - goto error; \ - } \ - PyObject *obj = PyUnicode_FromString(#FIELD); \ - if (obj == NULL) { \ - goto error; \ - } \ - ADD(#FIELD, obj); \ - } while (0) - - ADD_BOOL(use_main_obmalloc); - ADD_BOOL(allow_fork); - ADD_BOOL(allow_exec); - ADD_BOOL(allow_threads); - ADD_BOOL(allow_daemon_threads); - ADD_BOOL(check_multi_interp_extensions); - - ADD_STR(gil, gil_flag_to_str(config->gil)); - -#undef ADD_STR -#undef ADD_BOOL -#undef ADD - - return dict; - -error: - Py_DECREF(dict); - return NULL; -} - -static int -_config_dict_get_bool(PyObject *dict, const char *name, int *p_flag) -{ - PyObject *item; - if (_config_dict_get(dict, name, &item) < 0) { - return -1; - } - int flag = PyObject_IsTrue(item); - Py_DECREF(item); - if (flag < 0) { - return -1; - } - *p_flag = flag; - return 0; -} - -static int -_config_dict_copy_str(PyObject *dict, const char *name, - char *buf, size_t bufsize) -{ - PyObject *item; - if (_config_dict_get(dict, name, &item) < 0) { - return -1; - } - if (!PyUnicode_Check(item)) { - Py_DECREF(item); - config_dict_invalid_type(name); - return -1; - } - strncpy(buf, PyUnicode_AsUTF8(item), bufsize-1); - buf[bufsize-1] = '\0'; - Py_DECREF(item); - return 0; -} - -static int -interp_config_from_dict(PyObject *origdict, PyInterpreterConfig *config, - bool missing_allowed) -{ - PyObject *dict = PyDict_New(); - if (dict == NULL) { - return -1; - } - if (PyDict_Update(dict, origdict) < 0) { - goto error; - } - -#define CHECK(NAME) \ - do { \ - if (PyErr_Occurred()) { \ - goto error; \ - } \ - else { \ - if (!missing_allowed) { \ - (void)config_dict_get(dict, NAME); \ - assert(PyErr_Occurred()); \ - goto error; \ - } \ - } \ - } while (0) -#define COPY_BOOL(FIELD) \ - do { \ - int flag; \ - if (_config_dict_get_bool(dict, #FIELD, &flag) < 0) { \ - CHECK(#FIELD); \ - } \ - else { \ - config->FIELD = flag; \ - (void)PyDict_PopString(dict, #FIELD, NULL); \ - } \ - } while (0) - - COPY_BOOL(use_main_obmalloc); - COPY_BOOL(allow_fork); - COPY_BOOL(allow_exec); - COPY_BOOL(allow_threads); - COPY_BOOL(allow_daemon_threads); - COPY_BOOL(check_multi_interp_extensions); - - // PyInterpreterConfig.gil - char buf[20]; - if (_config_dict_copy_str(dict, "gil", buf, 20) < 0) { - CHECK("gil"); - } - else { - int flag; - if (gil_flag_from_str(buf, &flag) < 0) { - goto error; - } - config->gil = flag; - (void)PyDict_PopString(dict, "gil", NULL); - } - -#undef COPY_BOOL -#undef CHECK - - Py_ssize_t unused = PyDict_GET_SIZE(dict); - if (unused == 1) { - PyErr_Format(PyExc_ValueError, - "config dict has 1 extra item (%R)", dict); - goto error; - } - else if (unused > 0) { - PyErr_Format(PyExc_ValueError, - "config dict has %d extra items (%R)", unused, dict); - goto error; - } - return 0; - -error: - Py_DECREF(dict); - return -1; -} - -int -_PyInterpreterConfig_FromDict(PyObject *dict, PyInterpreterConfig *config) -{ - if (!PyDict_Check(dict)) { - PyErr_SetString(PyExc_TypeError, "dict expected"); - return -1; - } - if (interp_config_from_dict(dict, config, false) < 0) { - return -1; - } - return 0; -} - -int -_PyInterpreterConfig_UpdateFromDict(PyInterpreterConfig *config, PyObject *dict) -{ - if (!PyDict_Check(dict)) { - PyErr_SetString(PyExc_TypeError, "dict expected"); - return -1; - } - if (interp_config_from_dict(dict, config, true) < 0) { - return -1; - } - return 0; -} diff --git a/Python/interpconfig.c b/Python/interpconfig.c new file mode 100644 index 00000000000000..601fc6744567e8 --- /dev/null +++ b/Python/interpconfig.c @@ -0,0 +1,241 @@ +/* PyInterpreterConfig API */ + +#include "Python.h" +#include "pycore_pylifecycle.h" + +#include + +#include "config_common.h" + + +static const char * +gil_flag_to_str(int flag) +{ + switch (flag) { + case PyInterpreterConfig_DEFAULT_GIL: + return "default"; + case PyInterpreterConfig_SHARED_GIL: + return "shared"; + case PyInterpreterConfig_OWN_GIL: + return "own"; + default: + PyErr_SetString(PyExc_SystemError, + "invalid interpreter config 'gil' value"); + return NULL; + } +} + +static int +gil_flag_from_str(const char *str, int *p_flag) +{ + int flag; + if (str == NULL || strcmp(str, "") == 0) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "default") == 0) { + flag = PyInterpreterConfig_DEFAULT_GIL; + } + else if (strcmp(str, "shared") == 0) { + flag = PyInterpreterConfig_SHARED_GIL; + } + else if (strcmp(str, "own") == 0) { + flag = PyInterpreterConfig_OWN_GIL; + } + else { + PyErr_SetString(PyExc_SystemError, + "invalid interpreter config 'gil' value"); + return -1; + } + *p_flag = flag; + return 0; +} + +PyObject * +_PyInterpreterConfig_AsDict(PyInterpreterConfig *config) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + +#define ADD(NAME, OBJ) \ + do { \ + int res = PyDict_SetItemString(dict, NAME, (OBJ)); \ + Py_DECREF(OBJ); \ + if (res < 0) { \ + goto error; \ + } \ + } while (0) +#define ADD_BOOL(FIELD) \ + ADD(#FIELD, Py_NewRef(config->FIELD ? Py_True : Py_False)) +#define ADD_STR(FIELD, STR) \ + do { \ + if (STR == NULL) { \ + goto error; \ + } \ + PyObject *obj = PyUnicode_FromString(#FIELD); \ + if (obj == NULL) { \ + goto error; \ + } \ + ADD(#FIELD, obj); \ + } while (0) + + ADD_BOOL(use_main_obmalloc); + ADD_BOOL(allow_fork); + ADD_BOOL(allow_exec); + ADD_BOOL(allow_threads); + ADD_BOOL(allow_daemon_threads); + ADD_BOOL(check_multi_interp_extensions); + + ADD_STR(gil, gil_flag_to_str(config->gil)); + +#undef ADD_STR +#undef ADD_BOOL +#undef ADD + + return dict; + +error: + Py_DECREF(dict); + return NULL; +} + +static int +_config_dict_get_bool(PyObject *dict, const char *name, int *p_flag) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + int flag = PyObject_IsTrue(item); + Py_DECREF(item); + if (flag < 0) { + return -1; + } + *p_flag = flag; + return 0; +} + +static int +_config_dict_copy_str(PyObject *dict, const char *name, + char *buf, size_t bufsize) +{ + PyObject *item; + if (_config_dict_get(dict, name, &item) < 0) { + return -1; + } + if (!PyUnicode_Check(item)) { + Py_DECREF(item); + config_dict_invalid_type(name); + return -1; + } + strncpy(buf, PyUnicode_AsUTF8(item), bufsize-1); + buf[bufsize-1] = '\0'; + Py_DECREF(item); + return 0; +} + +static int +interp_config_from_dict(PyObject *origdict, PyInterpreterConfig *config, + bool missing_allowed) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + return -1; + } + if (PyDict_Update(dict, origdict) < 0) { + goto error; + } + +#define CHECK(NAME) \ + do { \ + if (PyErr_Occurred()) { \ + goto error; \ + } \ + else { \ + if (!missing_allowed) { \ + (void)config_dict_get(dict, NAME); \ + assert(PyErr_Occurred()); \ + goto error; \ + } \ + } \ + } while (0) +#define COPY_BOOL(FIELD) \ + do { \ + int flag; \ + if (_config_dict_get_bool(dict, #FIELD, &flag) < 0) { \ + CHECK(#FIELD); \ + } \ + else { \ + config->FIELD = flag; \ + (void)PyDict_PopString(dict, #FIELD, NULL); \ + } \ + } while (0) + + COPY_BOOL(use_main_obmalloc); + COPY_BOOL(allow_fork); + COPY_BOOL(allow_exec); + COPY_BOOL(allow_threads); + COPY_BOOL(allow_daemon_threads); + COPY_BOOL(check_multi_interp_extensions); + + // PyInterpreterConfig.gil + char buf[20]; + if (_config_dict_copy_str(dict, "gil", buf, 20) < 0) { + CHECK("gil"); + } + else { + int flag; + if (gil_flag_from_str(buf, &flag) < 0) { + goto error; + } + config->gil = flag; + (void)PyDict_PopString(dict, "gil", NULL); + } + +#undef COPY_BOOL +#undef CHECK + + Py_ssize_t unused = PyDict_GET_SIZE(dict); + if (unused == 1) { + PyErr_Format(PyExc_ValueError, + "config dict has 1 extra item (%R)", dict); + goto error; + } + else if (unused > 0) { + PyErr_Format(PyExc_ValueError, + "config dict has %d extra items (%R)", unused, dict); + goto error; + } + return 0; + +error: + Py_DECREF(dict); + return -1; +} + +int +_PyInterpreterConfig_FromDict(PyObject *dict, PyInterpreterConfig *config) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, false) < 0) { + return -1; + } + return 0; +} + +int +_PyInterpreterConfig_UpdateFromDict(PyInterpreterConfig *config, PyObject *dict) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + if (interp_config_from_dict(dict, config, true) < 0) { + return -1; + } + return 0; +} From a1130179235e677cd9c84f4de092b32d1a05d260 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 11:41:37 -0600 Subject: [PATCH 06/19] _PyInterpreterState_ResolveConfig() -> _PyInterpreterConfig_InitFromState() --- Include/internal/pycore_pylifecycle.h | 12 ++++++------ Modules/_testinternalcapi.c | 6 +++--- Python/interpconfig.c | 24 +++++++++++++++++++++++- Python/pystate.c | 22 ---------------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index a230cd2c5b927a..47ff0806574ac0 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -120,13 +120,13 @@ int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCo /* interpreter config */ // Export for _testinternalcapi shared extension -PyAPI_FUNC(int) _PyInterpreterState_ResolveConfig( - PyInterpreterState *, - PyInterpreterConfig *); +PyAPI_FUNC(int) _PyInterpreterConfig_InitFromState( + PyInterpreterConfig *, + PyInterpreterState *); PyAPI_FUNC(PyObject *) _PyInterpreterConfig_AsDict(PyInterpreterConfig *); -PyAPI_FUNC(int) _PyInterpreterConfig_FromDict( - PyObject *, - PyInterpreterConfig *); +PyAPI_FUNC(int) _PyInterpreterConfig_InitFromDict( + PyInterpreterConfig *, + PyObject *); PyAPI_FUNC(int) _PyInterpreterConfig_UpdateFromDict( PyInterpreterConfig *, PyObject *); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index aff6af9e47c75d..05bf0947cf74eb 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -28,7 +28,7 @@ #include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include "pycore_pylifecycle.h" // _PyInterpreterState_ResolveConfig() +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_InitFromState() #include "pycore_pystate.h" // _PyThreadState_GET() #include "clinic/_testinternalcapi.c.h" @@ -1399,7 +1399,7 @@ get_interpreter_config(PyObject *self, PyObject *args) } PyInterpreterConfig config; - if (_PyInterpreterState_ResolveConfig(interp, &config) < 0) { + if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { return NULL; } PyObject *dict = _PyInterpreterConfig_AsDict(&config); @@ -1484,7 +1484,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } PyInterpreterConfig config = {0}; - int res = _PyInterpreterConfig_FromDict(dict, &config); + int res = _PyInterpreterConfig_InitFromDict(&config, dict); Py_DECREF(dict); if (res < 0) { return NULL; diff --git a/Python/interpconfig.c b/Python/interpconfig.c index 601fc6744567e8..e866326fdee07a 100644 --- a/Python/interpconfig.c +++ b/Python/interpconfig.c @@ -215,7 +215,7 @@ interp_config_from_dict(PyObject *origdict, PyInterpreterConfig *config, } int -_PyInterpreterConfig_FromDict(PyObject *dict, PyInterpreterConfig *config) +_PyInterpreterConfig_InitFromDict(PyInterpreterConfig *config, PyObject *dict) { if (!PyDict_Check(dict)) { PyErr_SetString(PyExc_TypeError, "dict expected"); @@ -239,3 +239,25 @@ _PyInterpreterConfig_UpdateFromDict(PyInterpreterConfig *config, PyObject *dict) } return 0; } + +int +_PyInterpreterConfig_InitFromState(PyInterpreterConfig *config, + PyInterpreterState *interp) +{ + // Populate the config by re-constructing the values from the interpreter. + *config = (PyInterpreterConfig){ +#define FLAG(flag) \ + (interp->feature_flags & Py_RTFLAGS_ ## flag) + .use_main_obmalloc = FLAG(USE_MAIN_OBMALLOC), + .allow_fork = FLAG(FORK), + .allow_exec = FLAG(EXEC), + .allow_threads = FLAG(THREADS), + .allow_daemon_threads = FLAG(DAEMON_THREADS), + .check_multi_interp_extensions = FLAG(MULTI_INTERP_EXTENSIONS), +#undef FLAG + .gil = interp->ceval.own_gil + ? PyInterpreterConfig_OWN_GIL + : PyInterpreterConfig_SHARED_GIL, + }; + return 0; +} diff --git a/Python/pystate.c b/Python/pystate.c index 4a2106f19701c8..47d327ae28933b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1122,28 +1122,6 @@ PyInterpreterState_GetDict(PyInterpreterState *interp) return interp->dict; } -int -_PyInterpreterState_ResolveConfig(PyInterpreterState *interp, - PyInterpreterConfig *config) -{ - // Populate the config by re-constructing the values from the interpreter. - *config = (PyInterpreterConfig){ -#define FLAG(flag) \ - (interp->feature_flags & Py_RTFLAGS_ ## flag) - .use_main_obmalloc = FLAG(USE_MAIN_OBMALLOC), - .allow_fork = FLAG(FORK), - .allow_exec = FLAG(EXEC), - .allow_threads = FLAG(THREADS), - .allow_daemon_threads = FLAG(DAEMON_THREADS), - .check_multi_interp_extensions = FLAG(MULTI_INTERP_EXTENSIONS), -#undef FLAG - .gil = interp->ceval.own_gil - ? PyInterpreterConfig_OWN_GIL - : PyInterpreterConfig_SHARED_GIL, - }; - return 0; -} - //---------- // interp ID From fce72b818ae9c4c264df5c1bb9b2cc6b0ff0efd0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 16:12:27 -0600 Subject: [PATCH 07/19] _testinternalcapi.new_interpreter_config() -> _xxsubinterpreters.new_config() --- Lib/test/test_interpreters/test_api.py | 143 ++++++++++++++++++++++++- Lib/test/test_interpreters/utils.py | 21 +++- Modules/_testinternalcapi.c | 53 --------- Modules/_xxsubinterpretersmodule.c | 76 +++++++++++++ Python/interpconfig.c | 17 +-- 5 files changed, 246 insertions(+), 64 deletions(-) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 3cde9bd0014d9a..73b28f0877d2e9 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1,18 +1,27 @@ import os import pickle -import threading from textwrap import dedent +import threading +import types import unittest +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None from test import support from test.support import import_helper # Raise SkipTest if subinterpreters not supported. -import_helper.import_module('_xxsubinterpreters') +_interpreters = import_helper.import_module('_xxsubinterpreters') from test.support import interpreters from test.support.interpreters import InterpreterNotFoundError from .utils import _captured_script, _run_output, _running, TestBase +def requires__testinternalcapi(func): + return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func) + + class ModuleTests(TestBase): def test_queue_aliases(self): @@ -932,6 +941,136 @@ class SubBytes(bytes): interpreters.is_shareable(obj)) +class LowLevelTests(TestBase): + + # The behaviors in the low-level module are important in as much + # as they are exercised by the high-level module. Therefore the + # most important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + @requires__testinternalcapi + def test_new_config(self): + default = _interpreters.new_config('isolated') + with self.subTest('no arg'): + config = _interpreters.new_config() + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + with self.subTest('default'): + config1 = _interpreters.new_config('default') + self.assert_ns_equal(config1, default) + self.assertIsNot(config1, default) + + config2 = _interpreters.new_config('default') + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + for arg in ['', 'default']: + with self.subTest(f'default ({arg!r})'): + config = _interpreters.new_config(arg) + self.assert_ns_equal(config, default) + self.assertIsNot(config, default) + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=False, + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + for name, vanilla in supported.items(): + with self.subTest(f'supported ({name})'): + expected = vanilla + config1 = _interpreters.new_config(name) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _interpreters.new_config(name) + self.assert_ns_equal(config2, config1) + self.assertIsNot(config2, config1) + + with self.subTest(f'noop override ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'override all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _interpreters.new_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('extra override'): + with self.assertRaises(ValueError): + _interpreters.new_config(spam=True) + + # Bad values for bool fields. + for field, value in vars(supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'bad override ({field}={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(TypeError): + _interpreters.new_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'bad override (gil={value!r})'): + with self.assertRaises(ValueError): + _interpreters.new_config(gil=value) + + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. unittest.main() diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 973d05d4f96dcb..5ade6762ea24ef 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -68,6 +68,9 @@ def run(): class TestBase(unittest.TestCase): + def tearDown(self): + clean_up_interpreters() + def pipe(self): def ensure_closed(fd): try: @@ -156,5 +159,19 @@ def assert_python_failure(self, *argv): self.assertNotEqual(exitcode, 0) return stdout, stderr - def tearDown(self): - clean_up_interpreters() + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 05bf0947cf74eb..f7c44afacfa81f 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1412,57 +1412,6 @@ get_interpreter_config(PyObject *self, PyObject *args) return configobj; } -static PyObject * -new_interpreter_config(PyObject *self, PyObject *args, PyObject *kwargs) -{ - const char *initialized = NULL; - PyObject *overrides = NULL; - static char *kwlist[] = {"initialized", "overrides", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|sO:new_interpreter_config", kwlist, - &initialized, &overrides)) - { - return NULL; - } - if (initialized == NULL - || strcmp(initialized, "") == 0 - || strcmp(initialized, "default") == 0) - { - initialized = "isolated"; - } - - PyInterpreterConfig config; - if (strcmp(initialized, "isolated") == 0) { - config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; - } - else if (strcmp(initialized, "legacy") == 0) { - config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; - } - else if (strcmp(initialized, "empty") == 0) { - config = (PyInterpreterConfig){0}; - } - else { - PyErr_Format(PyExc_ValueError, - "unsupported initialized arg '%s'", initialized); - return NULL; - } - - if (overrides != NULL) { - if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { - return NULL; - } - } - - PyObject *dict = _PyInterpreterConfig_AsDict(&config); - if (dict == NULL) { - return NULL; - } - - PyObject *configobj = _PyNamespace_New(dict); - Py_DECREF(dict); - return configobj; -} - /* To run some code in a sub-interpreter. */ static PyObject * @@ -1903,8 +1852,6 @@ static PyMethodDef module_functions[] = { {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, {"get_interpreter_config", get_interpreter_config, METH_VARARGS}, - {"new_interpreter_config", _PyCFunction_CAST(new_interpreter_config), - METH_VARARGS | METH_KEYWORDS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index befa225c9183c5..a0dfdc7d52068a 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -12,8 +12,10 @@ #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() #include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_modsupport.h" // _PyArg_BadArgument() +#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_pyerrors.h" // _Py_excinfo +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #include "marshal.h" // PyMarshal_ReadObjectFromString() @@ -349,6 +351,34 @@ get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) /* interpreter-specific code ************************************************/ +static int +init_named_config(PyInterpreterConfig *config, const char *name) +{ + if (name == NULL + || strcmp(name, "") == 0 + || strcmp(name, "default") == 0) + { + name = "isolated"; + } + + if (strcmp(name, "isolated") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + } + else if (strcmp(name, "legacy") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + } + else if (strcmp(name, "empty") == 0) { + *config = (PyInterpreterConfig){0}; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported config name '%s'", name); + return -1; + } + return 0; +} + + static int _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) { @@ -417,6 +447,50 @@ _run_in_interpreter(PyInterpreterState *interp, /* module level code ********************************************************/ +static PyObject * +interp_new_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + const char *name = NULL; + if (!PyArg_ParseTuple(args, "|s:" MODULE_NAME_STR ".new_config", + &name)) + { + return NULL; + } + PyObject *overrides = kwds; + + PyInterpreterConfig config; + if (init_named_config(&config, name) < 0) { + return NULL; + } + + if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { + if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { + return NULL; + } + } + + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +PyDoc_STRVAR(new_config_doc, +"new_config(name='isolated', /, **overrides) -> type.SimpleNamespace\n\ +\n\ +Return a representation of a new PyInterpreterConfig.\n\ +\n\ +The name determines the initial values of the config. Supported named\n\ +configs are: default, isolated, legacy, and empty.\n\ +\n\ +Any keyword arguments are set on the corresponding config fields,\n\ +overriding the initial values."); + + static PyObject * interp_create(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1033,6 +1107,8 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds) static PyMethodDef module_functions[] = { + {"new_config", _PyCFunction_CAST(interp_new_config), + METH_VARARGS | METH_KEYWORDS, new_config_doc}, {"create", _PyCFunction_CAST(interp_create), METH_VARARGS | METH_KEYWORDS, create_doc}, {"destroy", _PyCFunction_CAST(interp_destroy), diff --git a/Python/interpconfig.c b/Python/interpconfig.c index e866326fdee07a..a4eb9d7ee9aebc 100644 --- a/Python/interpconfig.c +++ b/Python/interpconfig.c @@ -29,7 +29,7 @@ static int gil_flag_from_str(const char *str, int *p_flag) { int flag; - if (str == NULL || strcmp(str, "") == 0) { + if (str == NULL) { flag = PyInterpreterConfig_DEFAULT_GIL; } else if (strcmp(str, "default") == 0) { @@ -42,8 +42,8 @@ gil_flag_from_str(const char *str, int *p_flag) flag = PyInterpreterConfig_OWN_GIL; } else { - PyErr_SetString(PyExc_SystemError, - "invalid interpreter config 'gil' value"); + PyErr_Format(PyExc_ValueError, + "unsupported interpreter config .gil value '%s'", str); return -1; } *p_flag = flag; @@ -73,7 +73,7 @@ _PyInterpreterConfig_AsDict(PyInterpreterConfig *config) if (STR == NULL) { \ goto error; \ } \ - PyObject *obj = PyUnicode_FromString(#FIELD); \ + PyObject *obj = PyUnicode_FromString(STR); \ if (obj == NULL) { \ goto error; \ } \ @@ -107,11 +107,14 @@ _config_dict_get_bool(PyObject *dict, const char *name, int *p_flag) if (_config_dict_get(dict, name, &item) < 0) { return -1; } - int flag = PyObject_IsTrue(item); - Py_DECREF(item); - if (flag < 0) { + // For now we keep things strict, rather than using PyObject_IsTrue(). + int flag = item == Py_True; + if (!flag && item != Py_False) { + Py_DECREF(item); + config_dict_invalid_type(name); return -1; } + Py_DECREF(item); *p_flag = flag; return 0; } From c52e484172de0a7dd768f1db8402455d4578f524 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 15:40:56 -0600 Subject: [PATCH 08/19] _testinternalcapi.get_interpreter_config() -> _xxsubinterpreters.get_config() --- Lib/test/test_interpreters/test_api.py | 22 ++++++++++++- Modules/_testinternalcapi.c | 38 +--------------------- Modules/_xxsubinterpretersmodule.c | 44 ++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 38 deletions(-) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 73b28f0877d2e9..a584e0677c59e1 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -950,7 +950,6 @@ class LowLevelTests(TestBase): # encountered by the high-level module, thus they # mostly shouldn't matter as much. - @requires__testinternalcapi def test_new_config(self): default = _interpreters.new_config('isolated') with self.subTest('no arg'): @@ -1070,6 +1069,27 @@ def test_new_config(self): with self.assertRaises(ValueError): _interpreters.new_config(gil=value) + @requires__testinternalcapi + def test_get_config(self): + with self.subTest('main'): + expected = _interpreters.new_config('legacy') + expected.gil = 'own' + interpid = _interpreters.get_main() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _interpreters.new_config('isolated') + interpid = _interpreters.create(isolated=True) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _interpreters.new_config('legacy') + interpid = _interpreters.create(isolated=False) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index f7c44afacfa81f..fe3c1241309dec 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -23,12 +23,11 @@ #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() -#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include "pycore_pylifecycle.h" // _PyInterpreterConfig_InitFromState() +#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyThreadState_GET() #include "clinic/_testinternalcapi.c.h" @@ -1379,40 +1378,6 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } -static PyObject * -get_interpreter_config(PyObject *self, PyObject *args) -{ - PyObject *idobj = NULL; - if (!PyArg_ParseTuple(args, "|O:get_interpreter_config", &idobj)) { - return NULL; - } - - PyInterpreterState *interp; - if (idobj == NULL) { - interp = PyInterpreterState_Get(); - } - else { - interp = _PyInterpreterState_LookUpIDObject(idobj); - if (interp == NULL) { - return NULL; - } - } - - PyInterpreterConfig config; - if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { - return NULL; - } - PyObject *dict = _PyInterpreterConfig_AsDict(&config); - if (dict == NULL) { - return NULL; - } - - PyObject *configobj = _PyNamespace_New(dict); - Py_DECREF(dict); - return configobj; -} - - /* To run some code in a sub-interpreter. */ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) @@ -1851,7 +1816,6 @@ static PyMethodDef module_functions[] = { {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, - {"get_interpreter_config", get_interpreter_config, METH_VARARGS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index a0dfdc7d52068a..cd69701e0d6518 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1063,6 +1063,48 @@ PyDoc_STRVAR(is_running_doc, Return whether or not the identified interpreter is running."); +static PyObject * +interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *idobj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:get_config", kwlist, &idobj)) + { + return NULL; + } + + PyInterpreterState *interp; + if (idobj == NULL) { + interp = PyInterpreterState_Get(); + } + else { + interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + } + + PyInterpreterConfig config; + if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { + return NULL; + } + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +PyDoc_STRVAR(get_config_doc, +"get_config(id) -> types.SimpleNamespace\n\ +\n\ +Return a representation of the config used to initialize the interpreter."); + + static PyObject * interp_incref(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1122,6 +1164,8 @@ static PyMethodDef module_functions[] = { {"is_running", _PyCFunction_CAST(interp_is_running), METH_VARARGS | METH_KEYWORDS, is_running_doc}, + {"get_config", _PyCFunction_CAST(interp_get_config), + METH_VARARGS | METH_KEYWORDS, get_config_doc}, {"exec", _PyCFunction_CAST(interp_exec), METH_VARARGS | METH_KEYWORDS, exec_doc}, {"call", _PyCFunction_CAST(interp_call), From a2983ceda47c57cc5655625a35cddc7dcb3f630d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 14:53:58 -0600 Subject: [PATCH 09/19] Call _PyInterpreterState_RequireIDRef() in _interpreters._incref(). --- Modules/_xxsubinterpretersmodule.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index cd69701e0d6518..af0d944c78a271 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -540,7 +540,6 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) PyThreadState_Swap(save_tstate); PyThreadState_Delete(tstate); - _PyInterpreterState_RequireIDRef(interp, 1); return idobj; } @@ -1108,10 +1107,13 @@ Return a representation of the config used to initialize the interpreter."); static PyObject * interp_incref(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", NULL}; + static char *kwlist[] = {"id", "implieslink", NULL}; PyObject *id; + int implieslink = -1; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:_incref", kwlist, &id)) { + "O|$p:_incref", kwlist, + &id, &implieslink)) + { return NULL; } @@ -1119,6 +1121,14 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) if (interp == NULL) { return NULL; } + if (implieslink < 0) { + implieslink = !_Py_IsMainInterpreter(interp); + } + + if (implieslink) { + // Decref to 0 will destroy the interpreter. + _PyInterpreterState_RequireIDRef(interp, 1); + } if (_PyInterpreterState_IDInitref(interp) < 0) { return NULL; } From 05a081edecc3085324d8139e1ad28d55247d7292 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 19:05:31 -0600 Subject: [PATCH 10/19] _testinternalcapi.interpreter_incref() -> _interpreters._incref() --- Lib/test/test_capi/test_misc.py | 18 +++++++++--------- Modules/_testinternalcapi.c | 24 ------------------------ 2 files changed, 9 insertions(+), 33 deletions(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 55a1ab6d6d9359..c8f606e58b13d1 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2284,8 +2284,8 @@ def test_linked_lifecycle_does_not_exist(self): link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + incref = (lambda id: _interpreters._incref(id, implieslink=False)) + decref = _interpreters._decref with self.subTest('never existed'): interpid = _testinternalcapi.unused_interpreter_id() @@ -2339,8 +2339,8 @@ def test_linked_lifecycle_never_linked(self): exists = _testinternalcapi.interpreter_exists is_linked = _testinternalcapi.interpreter_refcount_linked get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + incref = (lambda id: _interpreters._incref(id, implieslink=False)) + decref = _interpreters._decref interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) @@ -2389,8 +2389,8 @@ def test_linked_lifecycle_link_incref_decref(self): is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + incref = (lambda id: _interpreters._incref(id, implieslink=False)) + decref = _interpreters._decref interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) @@ -2416,7 +2416,7 @@ def test_linked_lifecycle_incref_link(self): is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref + incref = (lambda id: _interpreters._incref(id, implieslink=False)) interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) @@ -2438,8 +2438,8 @@ def test_linked_lifecycle_link_incref_unlink_decref(self): link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = _testinternalcapi.interpreter_incref - decref = _testinternalcapi.interpreter_decref + incref = (lambda id: _interpreters._incref(id, implieslink=False)) + decref = _interpreters._decref interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index fe3c1241309dec..8a2baa266b3d9e 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1557,28 +1557,6 @@ interpreter_refcount_linked(PyObject *self, PyObject *idobj) Py_RETURN_FALSE; } -static PyObject * -interpreter_incref(PyObject *self, PyObject *idobj) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); - if (interp == NULL) { - return NULL; - } - _PyInterpreterState_IDIncref(interp); - Py_RETURN_NONE; -} - -static PyObject * -interpreter_decref(PyObject *self, PyObject *idobj) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); - if (interp == NULL) { - return NULL; - } - _PyInterpreterState_IDDecref(interp); - Py_RETURN_NONE; -} - static void _xid_capsule_destructor(PyObject *capsule) @@ -1827,8 +1805,6 @@ static PyMethodDef module_functions[] = { {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, {"interpreter_refcount_linked", interpreter_refcount_linked, METH_O}, - {"interpreter_incref", interpreter_incref, METH_O}, - {"interpreter_decref", interpreter_decref, METH_O}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, From 8a39bbc8a1b2add89d521367cd25da813c71498b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 14:22:12 -0600 Subject: [PATCH 11/19] Supporting passing a config to _xxsubinterpreters.create(). --- Lib/test/support/interpreters/__init__.py | 2 +- Lib/test/test__xxsubinterpreters.py | 4 +- Lib/test/test_importlib/test_util.py | 4 +- Lib/test/test_interpreters/test_api.py | 57 +++++++++++++++++++++-- Modules/_xxsubinterpretersmodule.c | 44 +++++++++++++---- 5 files changed, 95 insertions(+), 16 deletions(-) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index d8e6654fc96efd..f38ce1c9d84ae8 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -73,7 +73,7 @@ def __str__(self): def create(): """Return a new (idle) Python interpreter.""" - id = _interpreters.create(isolated=True) + id = _interpreters.create() return Interpreter(id) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 35d7355680e549..f674771c27cbb1 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -584,7 +584,7 @@ def f(): def test_create_daemon_thread(self): with self.subTest('isolated'): expected = 'spam spam spam spam spam' - subinterp = interpreters.create(isolated=True) + subinterp = interpreters.create('isolated') script, file = _captured_script(f""" import threading def f(): @@ -604,7 +604,7 @@ def f(): self.assertEqual(out, expected) with self.subTest('not isolated'): - subinterp = interpreters.create(isolated=False) + subinterp = interpreters.create('legacy') script, file = _captured_script(""" import threading def f(): diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index a6a76e589761e0..115cb7a56c98f7 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -656,7 +656,7 @@ def test_magic_number(self): class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase): def run_with_own_gil(self, script): - interpid = _interpreters.create(isolated=True) + interpid = _interpreters.create('isolated') def ensure_destroyed(): try: _interpreters.destroy(interpid) @@ -669,7 +669,7 @@ def ensure_destroyed(): raise ImportError(excsnap.msg) def run_with_shared_gil(self, script): - interpid = _interpreters.create(isolated=False) + interpid = _interpreters.create('legacy') def ensure_destroyed(): try: _interpreters.destroy(interpid) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index a584e0677c59e1..fed84a11f06084 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1069,7 +1069,6 @@ def test_new_config(self): with self.assertRaises(ValueError): _interpreters.new_config(gil=value) - @requires__testinternalcapi def test_get_config(self): with self.subTest('main'): expected = _interpreters.new_config('legacy') @@ -1080,16 +1079,68 @@ def test_get_config(self): with self.subTest('isolated'): expected = _interpreters.new_config('isolated') - interpid = _interpreters.create(isolated=True) + interpid = _interpreters.create('isolated') config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) with self.subTest('legacy'): expected = _interpreters.new_config('legacy') - interpid = _interpreters.create(isolated=False) + interpid = _interpreters.create('legacy') config = _interpreters.get_config(interpid) self.assert_ns_equal(config, expected) + @requires__testinternalcapi + def test_create(self): + isolated = _interpreters.new_config('isolated') + legacy = _interpreters.new_config('legacy') + default = isolated + + with self.subTest('no arg'): + interpid = _interpreters.create() + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('arg: None'): + interpid = _interpreters.create(None) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, default) + + with self.subTest('arg: \'empty\''): + with self.assertRaises(RuntimeError): + # The "empty" config isn't viable on its own. + _interpreters.create('empty') + + for arg, expected in { + '': default, + 'default': default, + 'isolated': isolated, + 'legacy': legacy, + }.items(): + with self.subTest(f'str arg: {arg!r}'): + interpid = _interpreters.create(arg) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _interpreters.new_config('empty') + orig.use_main_obmalloc = True + orig.gil = 'shared' + interpid = _interpreters.create(orig) + config = _interpreters.get_config(interpid) + self.assert_ns_equal(config, orig) + + with self.subTest('missing fields'): + orig = _interpreters.new_config() + del orig.gil + with self.assertRaises(ValueError): + _interpreters.create(orig) + + with self.subTest('extra fields'): + orig = _interpreters.new_config() + orig.spam = True + with self.assertRaises(ValueError): + _interpreters.create(orig) + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index af0d944c78a271..8d0fd72b63f489 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -378,6 +378,34 @@ init_named_config(PyInterpreterConfig *config, const char *name) return 0; } +static int +config_from_object(PyObject *configobj, PyInterpreterConfig *config) +{ + if (configobj == NULL || configobj == Py_None) { + if (init_named_config(config, NULL) < 0) { + return -1; + } + } + else if (PyUnicode_Check(configobj)) { + if (init_named_config(config, PyUnicode_AsUTF8(configobj)) < 0) { + return -1; + } + } + else { + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); + return -1; + } + int res = _PyInterpreterConfig_InitFromDict(config, dict); + Py_DECREF(dict); + if (res < 0) { + return -1; + } + } + return 0; +} + static int _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) @@ -494,21 +522,21 @@ overriding the initial values."); static PyObject * interp_create(PyObject *self, PyObject *args, PyObject *kwds) { + static char *kwlist[] = {"config", NULL}; + PyObject *configobj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:create", kwlist, + &configobj)) { + return NULL; + } - static char *kwlist[] = {"isolated", NULL}; - int isolated = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$i:create", kwlist, - &isolated)) { + PyInterpreterConfig config; + if (config_from_object(configobj, &config) < 0) { return NULL; } // Create and initialize the new interpreter. PyThreadState *save_tstate = PyThreadState_Get(); assert(save_tstate != NULL); - const PyInterpreterConfig config = isolated - ? (PyInterpreterConfig)_PyInterpreterConfig_INIT - : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; - // XXX Possible GILState issues? PyThreadState *tstate = NULL; PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); From 1173cd1cbb77b8abfe8d6d2079b9523788ec11c3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 22 Mar 2024 14:52:57 -0600 Subject: [PATCH 12/19] Factor out new_interpreter(). --- Lib/test/test_capi/test_misc.py | 12 ++--- Modules/_testinternalcapi.c | 45 ---------------- Modules/_xxsubinterpretersmodule.c | 87 ++++++++++++++++++++---------- 3 files changed, 65 insertions(+), 79 deletions(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index c8f606e58b13d1..4db2cbf7e9c2ff 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2327,7 +2327,7 @@ def test_linked_lifecycle_initial(self): get_refcount = _testinternalcapi.get_interpreter_refcount # A new interpreter will start out not linked, with a refcount of 0. - interpid = _testinternalcapi.new_interpreter() + interpid = self.new_interpreter() self.add_interp_cleanup(interpid) linked = is_linked(interpid) refcount = get_refcount(interpid) @@ -2342,7 +2342,7 @@ def test_linked_lifecycle_never_linked(self): incref = (lambda id: _interpreters._incref(id, implieslink=False)) decref = _interpreters._decref - interpid = _testinternalcapi.new_interpreter() + interpid = self.new_interpreter() self.add_interp_cleanup(interpid) # Incref will not automatically link it. @@ -2367,7 +2367,7 @@ def test_linked_lifecycle_link_unlink(self): link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount - interpid = _testinternalcapi.new_interpreter() + interpid = self.new_interpreter() self.add_interp_cleanup(interpid) # Linking at refcount 0 does not destroy the interpreter. @@ -2392,7 +2392,7 @@ def test_linked_lifecycle_link_incref_decref(self): incref = (lambda id: _interpreters._incref(id, implieslink=False)) decref = _interpreters._decref - interpid = _testinternalcapi.new_interpreter() + interpid = self.new_interpreter() self.add_interp_cleanup(interpid) # Linking it will not change the refcount. @@ -2418,7 +2418,7 @@ def test_linked_lifecycle_incref_link(self): get_refcount = _testinternalcapi.get_interpreter_refcount incref = (lambda id: _interpreters._incref(id, implieslink=False)) - interpid = _testinternalcapi.new_interpreter() + interpid = self.new_interpreter() self.add_interp_cleanup(interpid) incref(interpid) @@ -2441,7 +2441,7 @@ def test_linked_lifecycle_link_incref_unlink_decref(self): incref = (lambda id: _interpreters._incref(id, implieslink=False)) decref = _interpreters._decref - interpid = _testinternalcapi.new_interpreter() + interpid = self.new_interpreter() self.add_interp_cleanup(interpid) link(interpid) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 8a2baa266b3d9e..daac1b45e66589 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1451,50 +1451,6 @@ unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyLong_FromLongLong(interpid); } -static PyObject * -new_interpreter(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - // Unlike _interpreters.create(), we do not automatically link - // the interpreter to its refcount. - PyThreadState *save_tstate = PyThreadState_Get(); - const PyInterpreterConfig config = \ - (PyInterpreterConfig)_PyInterpreterConfig_INIT; - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - PyThreadState_Swap(save_tstate); - if (PyStatus_Exception(status)) { - _PyErr_SetFromPyStatus(status); - return NULL; - } - PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - - if (_PyInterpreterState_IDInitref(interp) < 0) { - goto error; - } - - int64_t interpid = PyInterpreterState_GetID(interp); - if (interpid < 0) { - goto error; - } - PyObject *idobj = PyLong_FromLongLong(interpid); - if (idobj == NULL) { - goto error; - } - - PyThreadState_Swap(tstate); - PyThreadState_Clear(tstate); - PyThreadState_Swap(save_tstate); - PyThreadState_Delete(tstate); - - return idobj; - -error: - save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); - return NULL; -} - static PyObject * interpreter_exists(PyObject *self, PyObject *idobj) { @@ -1799,7 +1755,6 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS}, {"normalize_interp_id", normalize_interp_id, METH_O}, {"unused_interpreter_id", unused_interpreter_id, METH_NOARGS}, - {"new_interpreter", new_interpreter, METH_NOARGS}, {"interpreter_exists", interpreter_exists, METH_O}, {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 8d0fd72b63f489..6cd3045711e31f 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -407,6 +407,59 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config) } +static PyInterpreterState * +new_interpreter(PyInterpreterConfig *config, PyObject **p_idobj, PyThreadState **p_tstate) +{ + PyThreadState *save_tstate = PyThreadState_Get(); + assert(save_tstate != NULL); + PyThreadState *tstate = NULL; + // XXX Possible GILState issues? + PyStatus status = Py_NewInterpreterFromConfig(&tstate, config); + PyThreadState_Swap(save_tstate); + if (PyStatus_Exception(status)) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + _PyErr_SetFromPyStatus(status); + return NULL; + } + assert(tstate != NULL); + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); + + if (_PyInterpreterState_IDInitref(interp) < 0) { + goto error; + } + + if (p_idobj != NULL) { + // We create the object using the original interpreter. + PyObject *idobj = get_interpid_obj(interp); + if (idobj == NULL) { + goto error; + } + *p_idobj = idobj; + } + + if (p_tstate != NULL) { + *p_tstate = tstate; + } + else { + PyThreadState_Swap(tstate); + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + } + + return interp; + +error: + // XXX Possible GILState issues? + save_tstate = PyThreadState_Swap(tstate); + Py_EndInterpreter(tstate); + PyThreadState_Swap(save_tstate); + return NULL; +} + + static int _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) { @@ -534,43 +587,21 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - // Create and initialize the new interpreter. - PyThreadState *save_tstate = PyThreadState_Get(); - assert(save_tstate != NULL); - // XXX Possible GILState issues? - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - PyThreadState_Swap(save_tstate); - if (PyStatus_Exception(status)) { - /* Since no new thread state was created, there is no exception to - propagate; raise a fresh one after swapping in the old thread - state. */ - _PyErr_SetFromPyStatus(status); + PyObject *idobj = NULL; + PyInterpreterState *interp = new_interpreter(&config, &idobj, NULL); + if (interp == NULL) { + // XXX Move the chained exception to interpreters.create()? PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); _PyErr_ChainExceptions1(exc); return NULL; } - assert(tstate != NULL); - - PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - PyObject *idobj = get_interpid_obj(interp); - if (idobj == NULL) { - // XXX Possible GILState issues? - save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); - return NULL; - } - - PyThreadState_Swap(tstate); - PyThreadState_Clear(tstate); - PyThreadState_Swap(save_tstate); - PyThreadState_Delete(tstate); return idobj; } + PyDoc_STRVAR(create_doc, "create() -> ID\n\ \n\ From 92c11d388baf7648f781937405c790370bc74674 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 25 Mar 2024 09:55:07 -0600 Subject: [PATCH 13/19] Fix test_import. --- Lib/test/test_import/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 81ec700d9755ce..3c387d973ce0f9 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2163,7 +2163,7 @@ def re_load(self, name, mod): # subinterpreters def add_subinterpreter(self): - interpid = _interpreters.create(isolated=False) + interpid = _interpreters.create('legacy') def ensure_destroyed(): try: _interpreters.destroy(interpid) From edda48dbe64ccc4d5e67f8eb60d0399935f59ea8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 25 Mar 2024 13:49:35 -0600 Subject: [PATCH 14/19] Fix an outdent. --- Python/interpconfig.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/interpconfig.c b/Python/interpconfig.c index a4eb9d7ee9aebc..419f40ae62a89e 100644 --- a/Python/interpconfig.c +++ b/Python/interpconfig.c @@ -73,7 +73,7 @@ _PyInterpreterConfig_AsDict(PyInterpreterConfig *config) if (STR == NULL) { \ goto error; \ } \ - PyObject *obj = PyUnicode_FromString(STR); \ + PyObject *obj = PyUnicode_FromString(STR); \ if (obj == NULL) { \ goto error; \ } \ From 5f617ed2ed7a4b5dfb451e5f3846fb72951cb116 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 1 Apr 2024 11:56:16 -0600 Subject: [PATCH 15/19] Call _PyInterpreterState_RequireIDRef() in the right places. --- Lib/test/support/interpreters/__init__.py | 2 +- Modules/_xxsubinterpretersmodule.c | 28 +++++++++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index f38ce1c9d84ae8..784724c1b94a6c 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -73,7 +73,7 @@ def __str__(self): def create(): """Return a new (idle) Python interpreter.""" - id = _interpreters.create() + id = _interpreters.create(reqrefs=True) return Interpreter(id) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 6cd3045711e31f..07d6e692bdb08b 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -575,10 +575,11 @@ overriding the initial values."); static PyObject * interp_create(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"config", NULL}; + static char *kwlist[] = {"config", "reqrefs", NULL}; PyObject *configobj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:create", kwlist, - &configobj)) { + int reqrefs = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O$p:create", kwlist, + &configobj, &reqrefs)) { return NULL; } @@ -598,16 +599,28 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } + if (reqrefs) { + // Decref to 0 will destroy the interpreter. + _PyInterpreterState_RequireIDRef(interp, 1); + } + return idobj; } PyDoc_STRVAR(create_doc, -"create() -> ID\n\ +"create([config], *, reqrefs=False) -> ID\n\ \n\ Create a new interpreter and return a unique generated ID.\n\ \n\ -The caller is responsible for destroying the interpreter before exiting."); +The caller is responsible for destroying the interpreter before exiting,\n\ +typically by using _interpreters.destroy(). This can be managed \n\ +automatically by passing \"reqrefs=True\" and then using _incref() and\n\ +_decref()` appropriately.\n\ +\n\ +\"config\" must be a valid interpreter config or the name of a\n\ +predefined config (\"isolated\" or \"legacy\"). The default\n\ +is \"isolated\"."); static PyObject * @@ -1168,7 +1181,7 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"id", "implieslink", NULL}; PyObject *id; - int implieslink = -1; + int implieslink = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|$p:_incref", kwlist, &id, &implieslink)) @@ -1180,9 +1193,6 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) if (interp == NULL) { return NULL; } - if (implieslink < 0) { - implieslink = !_Py_IsMainInterpreter(interp); - } if (implieslink) { // Decref to 0 will destroy the interpreter. From c504c79e3eccf8dac373fbf489c1da058691ef11 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 1 Apr 2024 11:57:23 -0600 Subject: [PATCH 16/19] Drop an unnecessary _PyInterpreterState_IDInitref() call. --- Modules/_xxsubinterpretersmodule.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 07d6e692bdb08b..f1a61f66abb4e7 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1198,9 +1198,6 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) // Decref to 0 will destroy the interpreter. _PyInterpreterState_RequireIDRef(interp, 1); } - if (_PyInterpreterState_IDInitref(interp) < 0) { - return NULL; - } _PyInterpreterState_IDIncref(interp); Py_RETURN_NONE; From 8a75c9002b0a1f52b64bb7ded652a4a6e48f4e58 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 2 Apr 2024 13:39:58 -0600 Subject: [PATCH 17/19] Reduce to just the new internal C-API. --- Lib/test/support/interpreters/__init__.py | 2 +- Lib/test/test__xxsubinterpreters.py | 4 +- Lib/test/test_capi/test_misc.py | 280 ++++++++++++++++++++-- Lib/test/test_import/__init__.py | 2 +- Lib/test/test_importlib/test_util.py | 4 +- Lib/test/test_interpreters/test_api.py | 214 +---------------- Lib/test/test_interpreters/test_queues.py | 6 +- Lib/test/test_interpreters/utils.py | 21 +- Modules/_testinternalcapi.c | 217 ++++++++++++++++- Modules/_xxsubinterpretersmodule.c | 278 ++++----------------- 10 files changed, 526 insertions(+), 502 deletions(-) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py index 784724c1b94a6c..d8e6654fc96efd 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/test/support/interpreters/__init__.py @@ -73,7 +73,7 @@ def __str__(self): def create(): """Return a new (idle) Python interpreter.""" - id = _interpreters.create(reqrefs=True) + id = _interpreters.create(isolated=True) return Interpreter(id) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index f674771c27cbb1..35d7355680e549 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -584,7 +584,7 @@ def f(): def test_create_daemon_thread(self): with self.subTest('isolated'): expected = 'spam spam spam spam spam' - subinterp = interpreters.create('isolated') + subinterp = interpreters.create(isolated=True) script, file = _captured_script(f""" import threading def f(): @@ -604,7 +604,7 @@ def f(): self.assertEqual(out, expected) with self.subTest('not isolated'): - subinterp = interpreters.create('legacy') + subinterp = interpreters.create(isolated=False) script, file = _captured_script(""" import threading def f(): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 4db2cbf7e9c2ff..a50f9c0f0dc822 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2204,6 +2204,256 @@ def test_module_state_shared_in_global(self): self.assertEqual(main_attr_id, subinterp_attr_id) +class InterpreterConfigTests(unittest.TestCase): + + supported = { + 'isolated': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=False, + check_multi_interp_extensions=True, + gil='own', + ), + 'legacy': types.SimpleNamespace( + use_main_obmalloc=True, + allow_fork=True, + allow_exec=True, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=False, + gil='shared', + ), + 'empty': types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=False, + allow_daemon_threads=False, + check_multi_interp_extensions=False, + gil='default', + ), + } + gil_supported = ['default', 'shared', 'own'] + + def iter_all_configs(self): + for use_main_obmalloc in (True, False): + for allow_fork in (True, False): + for allow_exec in (True, False): + for allow_threads in (True, False): + for allow_daemon in (True, False): + for checkext in (True, False): + for gil in ('shared', 'own', 'default'): + yield types.SimpleNamespace( + use_main_obmalloc=use_main_obmalloc, + allow_fork=allow_fork, + allow_exec=allow_exec, + allow_threads=allow_threads, + allow_daemon_threads=allow_daemon, + check_multi_interp_extensions=checkext, + gil=gil, + ) + + def assert_ns_equal(self, ns1, ns2, msg=None): + # This is mostly copied from TestCase.assertDictEqual. + self.assertEqual(type(ns1), type(ns2)) + if ns1 == ns2: + return + + import difflib + import pprint + from unittest.util import _common_shorten_repr + standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) + diff = ('\n' + '\n'.join(difflib.ndiff( + pprint.pformat(vars(ns1)).splitlines(), + pprint.pformat(vars(ns2)).splitlines()))) + diff = f'namespace({diff})' + standardMsg = self._truncateMessage(standardMsg, diff) + self.fail(self._formatMessage(msg, standardMsg)) + + def test_predefined_config(self): + def check(name, expected): + expected = self.supported[expected] + args = (name,) if name else () + + config1 = _testinternalcapi.new_interp_config(*args) + self.assert_ns_equal(config1, expected) + self.assertIsNot(config1, expected) + + config2 = _testinternalcapi.new_interp_config(*args) + self.assert_ns_equal(config2, expected) + self.assertIsNot(config2, expected) + self.assertIsNot(config2, config1) + + with self.subTest('default'): + check(None, 'isolated') + + for name in self.supported: + with self.subTest(name): + check(name, name) + + def test_update_from_dict(self): + for name, vanilla in self.supported.items(): + with self.subTest(f'noop ({name})'): + expected = vanilla + overrides = vars(vanilla) + config = _testinternalcapi.new_interp_config(name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest(f'change all ({name})'): + overrides = {k: not v for k, v in vars(vanilla).items()} + for gil in self.gil_supported: + if vanilla.gil == gil: + continue + overrides['gil'] = gil + expected = types.SimpleNamespace(**overrides) + config = _testinternalcapi.new_interp_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + # Override individual fields. + for field, old in vars(vanilla).items(): + if field == 'gil': + values = [v for v in self.gil_supported if v != old] + else: + values = [not old] + for val in values: + with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): + overrides = {field: val} + expected = types.SimpleNamespace( + **dict(vars(vanilla), **overrides), + ) + config = _testinternalcapi.new_interp_config( + name, **overrides) + self.assert_ns_equal(config, expected) + + with self.subTest('unsupported field'): + for name in self.supported: + with self.assertRaises(ValueError): + _testinternalcapi.new_interp_config(name, spam=True) + + # Bad values for bool fields. + for field, value in vars(self.supported['empty']).items(): + if field == 'gil': + continue + assert isinstance(value, bool) + for value in [1, '', 'spam', 1.0, None, object()]: + with self.subTest(f'unsupported value ({field}={value!r})'): + with self.assertRaises(TypeError): + _testinternalcapi.new_interp_config(**{field: value}) + + # Bad values for .gil. + for value in [True, 1, 1.0, None, object()]: + with self.subTest(f'unsupported value(gil={value!r})'): + with self.assertRaises(TypeError): + _testinternalcapi.new_interp_config(gil=value) + for value in ['', 'spam']: + with self.subTest(f'unsupported value (gil={value!r})'): + with self.assertRaises(ValueError): + _testinternalcapi.new_interp_config(gil=value) + + @requires_subinterpreters + def test_interp_init(self): + questionable = [ + # strange + dict( + allow_fork=True, + allow_exec=False, + ), + dict( + gil='shared', + use_main_obmalloc=False, + ), + # risky + dict( + allow_fork=True, + allow_threads=True, + ), + # ought to be invalid? + dict( + allow_threads=False, + allow_daemon_threads=True, + ), + dict( + gil='own', + use_main_obmalloc=True, + ), + ] + invalid = [ + dict( + use_main_obmalloc=False, + check_multi_interp_extensions=False + ), + ] + def match(config, override_cases): + ns = vars(config) + for overrides in override_cases: + if dict(ns, **overrides) == ns: + return True + return False + + def check(config): + script = 'pass' + rc = _testinternalcapi.run_in_subinterp_with_config(script, config) + self.assertEqual(rc, 0) + + for config in self.iter_all_configs(): + if config.gil == 'default': + continue + if match(config, invalid): + with self.subTest(f'invalid: {config}'): + with self.assertRaises(RuntimeError): + check(config) + elif match(config, questionable): + with self.subTest(f'questionable: {config}'): + check(config) + else: + with self.subTest(f'valid: {config}'): + check(config) + + @requires_subinterpreters + def test_get_config(self): + def new_interp(config): + interpid = _testinternalcapi.new_interpreter(config) + def ensure_destroyed(): + try: + _interpreters.destroy(interpid) + except _interpreters.InterpreterNotFoundError: + pass + self.addCleanup(ensure_destroyed) + return interpid + + with self.subTest('main'): + expected = _testinternalcapi.new_interp_config('legacy') + expected.gil = 'own' + interpid = _interpreters.get_main() + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('isolated'): + expected = _testinternalcapi.new_interp_config('isolated') + interpid = new_interp('isolated') + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('legacy'): + expected = _testinternalcapi.new_interp_config('legacy') + interpid = new_interp('legacy') + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, expected) + + with self.subTest('custom'): + orig = _testinternalcapi.new_interp_config( + 'empty', + use_main_obmalloc=True, + gil='shared', + ) + interpid = new_interp(orig) + config = _testinternalcapi.get_interp_config(interpid) + self.assert_ns_equal(config, orig) + + @requires_subinterpreters class InterpreterIDTests(unittest.TestCase): @@ -2284,8 +2534,8 @@ def test_linked_lifecycle_does_not_exist(self): link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = (lambda id: _interpreters._incref(id, implieslink=False)) - decref = _interpreters._decref + incref = _testinternalcapi.interpreter_incref + decref = _testinternalcapi.interpreter_decref with self.subTest('never existed'): interpid = _testinternalcapi.unused_interpreter_id() @@ -2327,7 +2577,7 @@ def test_linked_lifecycle_initial(self): get_refcount = _testinternalcapi.get_interpreter_refcount # A new interpreter will start out not linked, with a refcount of 0. - interpid = self.new_interpreter() + interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) linked = is_linked(interpid) refcount = get_refcount(interpid) @@ -2339,10 +2589,10 @@ def test_linked_lifecycle_never_linked(self): exists = _testinternalcapi.interpreter_exists is_linked = _testinternalcapi.interpreter_refcount_linked get_refcount = _testinternalcapi.get_interpreter_refcount - incref = (lambda id: _interpreters._incref(id, implieslink=False)) - decref = _interpreters._decref + incref = _testinternalcapi.interpreter_incref + decref = _testinternalcapi.interpreter_decref - interpid = self.new_interpreter() + interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) # Incref will not automatically link it. @@ -2367,7 +2617,7 @@ def test_linked_lifecycle_link_unlink(self): link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount - interpid = self.new_interpreter() + interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) # Linking at refcount 0 does not destroy the interpreter. @@ -2389,10 +2639,10 @@ def test_linked_lifecycle_link_incref_decref(self): is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = (lambda id: _interpreters._incref(id, implieslink=False)) - decref = _interpreters._decref + incref = _testinternalcapi.interpreter_incref + decref = _testinternalcapi.interpreter_decref - interpid = self.new_interpreter() + interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) # Linking it will not change the refcount. @@ -2416,9 +2666,9 @@ def test_linked_lifecycle_incref_link(self): is_linked = _testinternalcapi.interpreter_refcount_linked link = _testinternalcapi.link_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = (lambda id: _interpreters._incref(id, implieslink=False)) + incref = _testinternalcapi.interpreter_incref - interpid = self.new_interpreter() + interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) incref(interpid) @@ -2438,10 +2688,10 @@ def test_linked_lifecycle_link_incref_unlink_decref(self): link = _testinternalcapi.link_interpreter_refcount unlink = _testinternalcapi.unlink_interpreter_refcount get_refcount = _testinternalcapi.get_interpreter_refcount - incref = (lambda id: _interpreters._incref(id, implieslink=False)) - decref = _interpreters._decref + incref = _testinternalcapi.interpreter_incref + decref = _testinternalcapi.interpreter_decref - interpid = self.new_interpreter() + interpid = _testinternalcapi.new_interpreter() self.add_interp_cleanup(interpid) link(interpid) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 3c387d973ce0f9..81ec700d9755ce 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2163,7 +2163,7 @@ def re_load(self, name, mod): # subinterpreters def add_subinterpreter(self): - interpid = _interpreters.create('legacy') + interpid = _interpreters.create(isolated=False) def ensure_destroyed(): try: _interpreters.destroy(interpid) diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 115cb7a56c98f7..a6a76e589761e0 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -656,7 +656,7 @@ def test_magic_number(self): class IncompatibleExtensionModuleRestrictionsTests(unittest.TestCase): def run_with_own_gil(self, script): - interpid = _interpreters.create('isolated') + interpid = _interpreters.create(isolated=True) def ensure_destroyed(): try: _interpreters.destroy(interpid) @@ -669,7 +669,7 @@ def ensure_destroyed(): raise ImportError(excsnap.msg) def run_with_shared_gil(self, script): - interpid = _interpreters.create('legacy') + interpid = _interpreters.create(isolated=False) def ensure_destroyed(): try: _interpreters.destroy(interpid) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index fed84a11f06084..3cde9bd0014d9a 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1,27 +1,18 @@ import os import pickle -from textwrap import dedent import threading -import types +from textwrap import dedent import unittest -try: - import _testinternalcapi -except ImportError: - _testinternalcapi = None from test import support from test.support import import_helper # Raise SkipTest if subinterpreters not supported. -_interpreters = import_helper.import_module('_xxsubinterpreters') +import_helper.import_module('_xxsubinterpreters') from test.support import interpreters from test.support.interpreters import InterpreterNotFoundError from .utils import _captured_script, _run_output, _running, TestBase -def requires__testinternalcapi(func): - return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func) - - class ModuleTests(TestBase): def test_queue_aliases(self): @@ -941,207 +932,6 @@ class SubBytes(bytes): interpreters.is_shareable(obj)) -class LowLevelTests(TestBase): - - # The behaviors in the low-level module are important in as much - # as they are exercised by the high-level module. Therefore the - # most important testing happens in the high-level tests. - # These low-level tests cover corner cases that are not - # encountered by the high-level module, thus they - # mostly shouldn't matter as much. - - def test_new_config(self): - default = _interpreters.new_config('isolated') - with self.subTest('no arg'): - config = _interpreters.new_config() - self.assert_ns_equal(config, default) - self.assertIsNot(config, default) - - with self.subTest('default'): - config1 = _interpreters.new_config('default') - self.assert_ns_equal(config1, default) - self.assertIsNot(config1, default) - - config2 = _interpreters.new_config('default') - self.assert_ns_equal(config2, config1) - self.assertIsNot(config2, config1) - - for arg in ['', 'default']: - with self.subTest(f'default ({arg!r})'): - config = _interpreters.new_config(arg) - self.assert_ns_equal(config, default) - self.assertIsNot(config, default) - - supported = { - 'isolated': types.SimpleNamespace( - use_main_obmalloc=False, - allow_fork=False, - allow_exec=False, - allow_threads=True, - allow_daemon_threads=False, - check_multi_interp_extensions=True, - gil='own', - ), - 'legacy': types.SimpleNamespace( - use_main_obmalloc=True, - allow_fork=True, - allow_exec=True, - allow_threads=True, - allow_daemon_threads=True, - check_multi_interp_extensions=False, - gil='shared', - ), - 'empty': types.SimpleNamespace( - use_main_obmalloc=False, - allow_fork=False, - allow_exec=False, - allow_threads=False, - allow_daemon_threads=False, - check_multi_interp_extensions=False, - gil='default', - ), - } - gil_supported = ['default', 'shared', 'own'] - - for name, vanilla in supported.items(): - with self.subTest(f'supported ({name})'): - expected = vanilla - config1 = _interpreters.new_config(name) - self.assert_ns_equal(config1, expected) - self.assertIsNot(config1, expected) - - config2 = _interpreters.new_config(name) - self.assert_ns_equal(config2, config1) - self.assertIsNot(config2, config1) - - with self.subTest(f'noop override ({name})'): - expected = vanilla - overrides = vars(vanilla) - config = _interpreters.new_config(name, **overrides) - self.assert_ns_equal(config, expected) - - with self.subTest(f'override all ({name})'): - overrides = {k: not v for k, v in vars(vanilla).items()} - for gil in gil_supported: - if vanilla.gil == gil: - continue - overrides['gil'] = gil - expected = types.SimpleNamespace(**overrides) - config = _interpreters.new_config(name, **overrides) - self.assert_ns_equal(config, expected) - - # Override individual fields. - for field, old in vars(vanilla).items(): - if field == 'gil': - values = [v for v in gil_supported if v != old] - else: - values = [not old] - for val in values: - with self.subTest(f'{name}.{field} ({old!r} -> {val!r})'): - overrides = {field: val} - expected = types.SimpleNamespace( - **dict(vars(vanilla), **overrides), - ) - config = _interpreters.new_config(name, **overrides) - self.assert_ns_equal(config, expected) - - with self.subTest('extra override'): - with self.assertRaises(ValueError): - _interpreters.new_config(spam=True) - - # Bad values for bool fields. - for field, value in vars(supported['empty']).items(): - if field == 'gil': - continue - assert isinstance(value, bool) - for value in [1, '', 'spam', 1.0, None, object()]: - with self.subTest(f'bad override ({field}={value!r})'): - with self.assertRaises(TypeError): - _interpreters.new_config(**{field: value}) - - # Bad values for .gil. - for value in [True, 1, 1.0, None, object()]: - with self.subTest(f'bad override (gil={value!r})'): - with self.assertRaises(TypeError): - _interpreters.new_config(gil=value) - for value in ['', 'spam']: - with self.subTest(f'bad override (gil={value!r})'): - with self.assertRaises(ValueError): - _interpreters.new_config(gil=value) - - def test_get_config(self): - with self.subTest('main'): - expected = _interpreters.new_config('legacy') - expected.gil = 'own' - interpid = _interpreters.get_main() - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, expected) - - with self.subTest('isolated'): - expected = _interpreters.new_config('isolated') - interpid = _interpreters.create('isolated') - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, expected) - - with self.subTest('legacy'): - expected = _interpreters.new_config('legacy') - interpid = _interpreters.create('legacy') - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, expected) - - @requires__testinternalcapi - def test_create(self): - isolated = _interpreters.new_config('isolated') - legacy = _interpreters.new_config('legacy') - default = isolated - - with self.subTest('no arg'): - interpid = _interpreters.create() - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, default) - - with self.subTest('arg: None'): - interpid = _interpreters.create(None) - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, default) - - with self.subTest('arg: \'empty\''): - with self.assertRaises(RuntimeError): - # The "empty" config isn't viable on its own. - _interpreters.create('empty') - - for arg, expected in { - '': default, - 'default': default, - 'isolated': isolated, - 'legacy': legacy, - }.items(): - with self.subTest(f'str arg: {arg!r}'): - interpid = _interpreters.create(arg) - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, expected) - - with self.subTest('custom'): - orig = _interpreters.new_config('empty') - orig.use_main_obmalloc = True - orig.gil = 'shared' - interpid = _interpreters.create(orig) - config = _interpreters.get_config(interpid) - self.assert_ns_equal(config, orig) - - with self.subTest('missing fields'): - orig = _interpreters.new_config() - del orig.gil - with self.assertRaises(ValueError): - _interpreters.create(orig) - - with self.subTest('extra fields'): - orig = _interpreters.new_config() - orig.spam = True - with self.assertRaises(ValueError): - _interpreters.create(orig) - - if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. unittest.main() diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 8ab9ebb354712a..d16d294b82d044 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -28,9 +28,9 @@ def tearDown(self): class LowLevelTests(TestBase): - # The behaviors in the low-level module are important in as much - # as they are exercised by the high-level module. Therefore the - # most important testing happens in the high-level tests. + # The behaviors in the low-level module is important in as much + # as it is exercised by the high-level module. Therefore the + # most # important testing happens in the high-level tests. # These low-level tests cover corner cases that are not # encountered by the high-level module, thus they # mostly shouldn't matter as much. diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index 5ade6762ea24ef..973d05d4f96dcb 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -68,9 +68,6 @@ def run(): class TestBase(unittest.TestCase): - def tearDown(self): - clean_up_interpreters() - def pipe(self): def ensure_closed(fd): try: @@ -159,19 +156,5 @@ def assert_python_failure(self, *argv): self.assertNotEqual(exitcode, 0) return stdout, stderr - def assert_ns_equal(self, ns1, ns2, msg=None): - # This is mostly copied from TestCase.assertDictEqual. - self.assertEqual(type(ns1), type(ns2)) - if ns1 == ns2: - return - - import difflib - import pprint - from unittest.util import _common_shorten_repr - standardMsg = '%s != %s' % _common_shorten_repr(ns1, ns2) - diff = ('\n' + '\n'.join(difflib.ndiff( - pprint.pformat(vars(ns1)).splitlines(), - pprint.pformat(vars(ns2)).splitlines()))) - diff = f'namespace({diff})' - standardMsg = self._truncateMessage(standardMsg, diff) - self.fail(self._formatMessage(msg, standardMsg)) + def tearDown(self): + clean_up_interpreters() diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index daac1b45e66589..685193194ff9b6 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -23,6 +23,7 @@ #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() +#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_optimizer.h" // _Py_UopsSymbol, etc. #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() @@ -830,7 +831,6 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, } -// Maybe this could be replaced by get_interpreter_config()? static PyObject * get_interp_settings(PyObject *self, PyObject *args) { @@ -1378,6 +1378,129 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) } +static int +init_named_interp_config(PyInterpreterConfig *config, const char *name) +{ + if (name == NULL) { + name = "isolated"; + } + + if (strcmp(name, "isolated") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; + } + else if (strcmp(name, "legacy") == 0) { + *config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + } + else if (strcmp(name, "empty") == 0) { + *config = (PyInterpreterConfig){0}; + } + else { + PyErr_Format(PyExc_ValueError, + "unsupported config name '%s'", name); + return -1; + } + return 0; +} + +static PyObject * +new_interp_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + const char *name = NULL; + if (!PyArg_ParseTuple(args, "|s:new_config", &name)) { + return NULL; + } + PyObject *overrides = kwds; + + if (name == NULL) { + name = "isolated"; + } + + PyInterpreterConfig config; + if (init_named_interp_config(&config, name) < 0) { + return NULL; + } + + if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { + if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { + return NULL; + } + } + + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +static PyObject * +get_interp_config(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", NULL}; + PyObject *idobj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O:get_config", kwlist, &idobj)) + { + return NULL; + } + + PyInterpreterState *interp; + if (idobj == NULL) { + interp = PyInterpreterState_Get(); + } + else { + interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + } + + PyInterpreterConfig config; + if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { + return NULL; + } + PyObject *dict = _PyInterpreterConfig_AsDict(&config); + if (dict == NULL) { + return NULL; + } + + PyObject *configobj = _PyNamespace_New(dict); + Py_DECREF(dict); + return configobj; +} + +static int +interp_config_from_object(PyObject *configobj, PyInterpreterConfig *config) +{ + if (configobj == NULL || configobj == Py_None) { + if (init_named_interp_config(config, NULL) < 0) { + return -1; + } + } + else if (PyUnicode_Check(configobj)) { + if (init_named_interp_config(config, PyUnicode_AsUTF8(configobj)) < 0) { + return -1; + } + } + else { + PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); + if (dict == NULL) { + PyErr_Format(PyExc_TypeError, "bad config %R", configobj); + return -1; + } + int res = _PyInterpreterConfig_InitFromDict(config, dict); + Py_DECREF(dict); + if (res < 0) { + return -1; + } + } + return 0; +} + + /* To run some code in a sub-interpreter. */ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) @@ -1392,15 +1515,8 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } - PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); - if (dict == NULL) { - PyErr_Format(PyExc_TypeError, "bad config %R", configobj); - return NULL; - } - PyInterpreterConfig config = {0}; - int res = _PyInterpreterConfig_InitFromDict(&config, dict); - Py_DECREF(dict); - if (res < 0) { + PyInterpreterConfig config; + if (interp_config_from_object(configobj, &config) < 0) { return NULL; } @@ -1451,6 +1567,58 @@ unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyLong_FromLongLong(interpid); } +static PyObject * +new_interpreter(PyObject *self, PyObject *args) +{ + PyObject *configobj = NULL; + if (!PyArg_ParseTuple(args, "|O:new_interpreter", &configobj)) { + return NULL; + } + + PyInterpreterConfig config; + if (interp_config_from_object(configobj, &config) < 0) { + return NULL; + } + + // Unlike _interpreters.create(), we do not automatically link + // the interpreter to its refcount. + PyThreadState *save_tstate = PyThreadState_Get(); + PyThreadState *tstate = NULL; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); + PyThreadState_Swap(save_tstate); + if (PyStatus_Exception(status)) { + _PyErr_SetFromPyStatus(status); + return NULL; + } + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); + + if (_PyInterpreterState_IDInitref(interp) < 0) { + goto error; + } + + int64_t interpid = PyInterpreterState_GetID(interp); + if (interpid < 0) { + goto error; + } + PyObject *idobj = PyLong_FromLongLong(interpid); + if (idobj == NULL) { + goto error; + } + + PyThreadState_Swap(tstate); + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + + return idobj; + +error: + save_tstate = PyThreadState_Swap(tstate); + Py_EndInterpreter(tstate); + PyThreadState_Swap(save_tstate); + return NULL; +} + static PyObject * interpreter_exists(PyObject *self, PyObject *idobj) { @@ -1513,6 +1681,28 @@ interpreter_refcount_linked(PyObject *self, PyObject *idobj) Py_RETURN_FALSE; } +static PyObject * +interpreter_incref(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + _PyInterpreterState_IDIncref(interp); + Py_RETURN_NONE; +} + +static PyObject * +interpreter_decref(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + _PyInterpreterState_IDDecref(interp); + Py_RETURN_NONE; +} + static void _xid_capsule_destructor(PyObject *capsule) @@ -1750,16 +1940,23 @@ static PyMethodDef module_functions[] = { {"get_object_dict_values", get_object_dict_values, METH_O}, {"hamt", new_hamt, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, + {"new_interp_config", _PyCFunction_CAST(new_interp_config), + METH_VARARGS | METH_KEYWORDS}, + {"get_interp_config", _PyCFunction_CAST(get_interp_config), + METH_VARARGS | METH_KEYWORDS}, {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, {"normalize_interp_id", normalize_interp_id, METH_O}, {"unused_interpreter_id", unused_interpreter_id, METH_NOARGS}, + {"new_interpreter", new_interpreter, METH_VARARGS}, {"interpreter_exists", interpreter_exists, METH_O}, {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, {"interpreter_refcount_linked", interpreter_refcount_linked, METH_O}, + {"interpreter_incref", interpreter_incref, METH_O}, + {"interpreter_decref", interpreter_decref, METH_O}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index f1a61f66abb4e7..befa225c9183c5 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -12,10 +12,8 @@ #include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() #include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_modsupport.h" // _PyArg_BadArgument() -#include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree() #include "pycore_pyerrors.h" // _Py_excinfo -#include "pycore_pylifecycle.h" // _PyInterpreterConfig_AsDict() #include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #include "marshal.h" // PyMarshal_ReadObjectFromString() @@ -351,115 +349,6 @@ get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) /* interpreter-specific code ************************************************/ -static int -init_named_config(PyInterpreterConfig *config, const char *name) -{ - if (name == NULL - || strcmp(name, "") == 0 - || strcmp(name, "default") == 0) - { - name = "isolated"; - } - - if (strcmp(name, "isolated") == 0) { - *config = (PyInterpreterConfig)_PyInterpreterConfig_INIT; - } - else if (strcmp(name, "legacy") == 0) { - *config = (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; - } - else if (strcmp(name, "empty") == 0) { - *config = (PyInterpreterConfig){0}; - } - else { - PyErr_Format(PyExc_ValueError, - "unsupported config name '%s'", name); - return -1; - } - return 0; -} - -static int -config_from_object(PyObject *configobj, PyInterpreterConfig *config) -{ - if (configobj == NULL || configobj == Py_None) { - if (init_named_config(config, NULL) < 0) { - return -1; - } - } - else if (PyUnicode_Check(configobj)) { - if (init_named_config(config, PyUnicode_AsUTF8(configobj)) < 0) { - return -1; - } - } - else { - PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); - if (dict == NULL) { - PyErr_Format(PyExc_TypeError, "bad config %R", configobj); - return -1; - } - int res = _PyInterpreterConfig_InitFromDict(config, dict); - Py_DECREF(dict); - if (res < 0) { - return -1; - } - } - return 0; -} - - -static PyInterpreterState * -new_interpreter(PyInterpreterConfig *config, PyObject **p_idobj, PyThreadState **p_tstate) -{ - PyThreadState *save_tstate = PyThreadState_Get(); - assert(save_tstate != NULL); - PyThreadState *tstate = NULL; - // XXX Possible GILState issues? - PyStatus status = Py_NewInterpreterFromConfig(&tstate, config); - PyThreadState_Swap(save_tstate); - if (PyStatus_Exception(status)) { - /* Since no new thread state was created, there is no exception to - propagate; raise a fresh one after swapping in the old thread - state. */ - _PyErr_SetFromPyStatus(status); - return NULL; - } - assert(tstate != NULL); - PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); - - if (_PyInterpreterState_IDInitref(interp) < 0) { - goto error; - } - - if (p_idobj != NULL) { - // We create the object using the original interpreter. - PyObject *idobj = get_interpid_obj(interp); - if (idobj == NULL) { - goto error; - } - *p_idobj = idobj; - } - - if (p_tstate != NULL) { - *p_tstate = tstate; - } - else { - PyThreadState_Swap(tstate); - PyThreadState_Clear(tstate); - PyThreadState_Swap(save_tstate); - PyThreadState_Delete(tstate); - } - - return interp; - -error: - // XXX Possible GILState issues? - save_tstate = PyThreadState_Swap(tstate); - Py_EndInterpreter(tstate); - PyThreadState_Swap(save_tstate); - return NULL; -} - - static int _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) { @@ -528,99 +417,65 @@ _run_in_interpreter(PyInterpreterState *interp, /* module level code ********************************************************/ -static PyObject * -interp_new_config(PyObject *self, PyObject *args, PyObject *kwds) -{ - const char *name = NULL; - if (!PyArg_ParseTuple(args, "|s:" MODULE_NAME_STR ".new_config", - &name)) - { - return NULL; - } - PyObject *overrides = kwds; - - PyInterpreterConfig config; - if (init_named_config(&config, name) < 0) { - return NULL; - } - - if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { - if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { - return NULL; - } - } - - PyObject *dict = _PyInterpreterConfig_AsDict(&config); - if (dict == NULL) { - return NULL; - } - - PyObject *configobj = _PyNamespace_New(dict); - Py_DECREF(dict); - return configobj; -} - -PyDoc_STRVAR(new_config_doc, -"new_config(name='isolated', /, **overrides) -> type.SimpleNamespace\n\ -\n\ -Return a representation of a new PyInterpreterConfig.\n\ -\n\ -The name determines the initial values of the config. Supported named\n\ -configs are: default, isolated, legacy, and empty.\n\ -\n\ -Any keyword arguments are set on the corresponding config fields,\n\ -overriding the initial values."); - - static PyObject * interp_create(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"config", "reqrefs", NULL}; - PyObject *configobj = NULL; - int reqrefs = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O$p:create", kwlist, - &configobj, &reqrefs)) { - return NULL; - } - PyInterpreterConfig config; - if (config_from_object(configobj, &config) < 0) { + static char *kwlist[] = {"isolated", NULL}; + int isolated = 1; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|$i:create", kwlist, + &isolated)) { return NULL; } - PyObject *idobj = NULL; - PyInterpreterState *interp = new_interpreter(&config, &idobj, NULL); - if (interp == NULL) { - // XXX Move the chained exception to interpreters.create()? + // Create and initialize the new interpreter. + PyThreadState *save_tstate = PyThreadState_Get(); + assert(save_tstate != NULL); + const PyInterpreterConfig config = isolated + ? (PyInterpreterConfig)_PyInterpreterConfig_INIT + : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + + // XXX Possible GILState issues? + PyThreadState *tstate = NULL; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); + PyThreadState_Swap(save_tstate); + if (PyStatus_Exception(status)) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + _PyErr_SetFromPyStatus(status); PyObject *exc = PyErr_GetRaisedException(); - assert(exc != NULL); PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); _PyErr_ChainExceptions1(exc); return NULL; } + assert(tstate != NULL); - if (reqrefs) { - // Decref to 0 will destroy the interpreter. - _PyInterpreterState_RequireIDRef(interp, 1); + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); + PyObject *idobj = get_interpid_obj(interp); + if (idobj == NULL) { + // XXX Possible GILState issues? + save_tstate = PyThreadState_Swap(tstate); + Py_EndInterpreter(tstate); + PyThreadState_Swap(save_tstate); + return NULL; } + PyThreadState_Swap(tstate); + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + + _PyInterpreterState_RequireIDRef(interp, 1); return idobj; } - PyDoc_STRVAR(create_doc, -"create([config], *, reqrefs=False) -> ID\n\ +"create() -> ID\n\ \n\ Create a new interpreter and return a unique generated ID.\n\ \n\ -The caller is responsible for destroying the interpreter before exiting,\n\ -typically by using _interpreters.destroy(). This can be managed \n\ -automatically by passing \"reqrefs=True\" and then using _incref() and\n\ -_decref()` appropriately.\n\ -\n\ -\"config\" must be a valid interpreter config or the name of a\n\ -predefined config (\"isolated\" or \"legacy\"). The default\n\ -is \"isolated\"."); +The caller is responsible for destroying the interpreter before exiting."); static PyObject * @@ -1134,58 +989,13 @@ PyDoc_STRVAR(is_running_doc, Return whether or not the identified interpreter is running."); -static PyObject * -interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"id", NULL}; - PyObject *idobj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O:get_config", kwlist, &idobj)) - { - return NULL; - } - - PyInterpreterState *interp; - if (idobj == NULL) { - interp = PyInterpreterState_Get(); - } - else { - interp = _PyInterpreterState_LookUpIDObject(idobj); - if (interp == NULL) { - return NULL; - } - } - - PyInterpreterConfig config; - if (_PyInterpreterConfig_InitFromState(&config, interp) < 0) { - return NULL; - } - PyObject *dict = _PyInterpreterConfig_AsDict(&config); - if (dict == NULL) { - return NULL; - } - - PyObject *configobj = _PyNamespace_New(dict); - Py_DECREF(dict); - return configobj; -} - -PyDoc_STRVAR(get_config_doc, -"get_config(id) -> types.SimpleNamespace\n\ -\n\ -Return a representation of the config used to initialize the interpreter."); - - static PyObject * interp_incref(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"id", "implieslink", NULL}; + static char *kwlist[] = {"id", NULL}; PyObject *id; - int implieslink = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|$p:_incref", kwlist, - &id, &implieslink)) - { + "O:_incref", kwlist, &id)) { return NULL; } @@ -1193,10 +1003,8 @@ interp_incref(PyObject *self, PyObject *args, PyObject *kwds) if (interp == NULL) { return NULL; } - - if (implieslink) { - // Decref to 0 will destroy the interpreter. - _PyInterpreterState_RequireIDRef(interp, 1); + if (_PyInterpreterState_IDInitref(interp) < 0) { + return NULL; } _PyInterpreterState_IDIncref(interp); @@ -1225,8 +1033,6 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds) static PyMethodDef module_functions[] = { - {"new_config", _PyCFunction_CAST(interp_new_config), - METH_VARARGS | METH_KEYWORDS, new_config_doc}, {"create", _PyCFunction_CAST(interp_create), METH_VARARGS | METH_KEYWORDS, create_doc}, {"destroy", _PyCFunction_CAST(interp_destroy), @@ -1240,8 +1046,6 @@ static PyMethodDef module_functions[] = { {"is_running", _PyCFunction_CAST(interp_is_running), METH_VARARGS | METH_KEYWORDS, is_running_doc}, - {"get_config", _PyCFunction_CAST(interp_get_config), - METH_VARARGS | METH_KEYWORDS, get_config_doc}, {"exec", _PyCFunction_CAST(interp_exec), METH_VARARGS | METH_KEYWORDS, exec_doc}, {"call", _PyCFunction_CAST(interp_call), From a38cda7b9a3b7889404b1299ff5db6126260cd25 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 2 Apr 2024 14:00:45 -0600 Subject: [PATCH 18/19] Adjust test_get_config. --- Lib/test/test_capi/test_misc.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index a50f9c0f0dc822..34311afc93fc29 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2414,15 +2414,16 @@ def check(config): @requires_subinterpreters def test_get_config(self): + @contextlib.contextmanager def new_interp(config): interpid = _testinternalcapi.new_interpreter(config) - def ensure_destroyed(): + try: + yield interpid + finally: try: _interpreters.destroy(interpid) except _interpreters.InterpreterNotFoundError: pass - self.addCleanup(ensure_destroyed) - return interpid with self.subTest('main'): expected = _testinternalcapi.new_interp_config('legacy') @@ -2433,14 +2434,14 @@ def ensure_destroyed(): with self.subTest('isolated'): expected = _testinternalcapi.new_interp_config('isolated') - interpid = new_interp('isolated') - config = _testinternalcapi.get_interp_config(interpid) + with new_interp('isolated') as interpid: + config = _testinternalcapi.get_interp_config(interpid) self.assert_ns_equal(config, expected) with self.subTest('legacy'): expected = _testinternalcapi.new_interp_config('legacy') - interpid = new_interp('legacy') - config = _testinternalcapi.get_interp_config(interpid) + with new_interp('legacy') as interpid: + config = _testinternalcapi.get_interp_config(interpid) self.assert_ns_equal(config, expected) with self.subTest('custom'): @@ -2449,8 +2450,8 @@ def ensure_destroyed(): use_main_obmalloc=True, gil='shared', ) - interpid = new_interp(orig) - config = _testinternalcapi.get_interp_config(interpid) + with new_interp(orig) as interpid: + config = _testinternalcapi.get_interp_config(interpid) self.assert_ns_equal(config, orig) From cae0482fb30925440202068823703ac78efc2974 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 2 Apr 2024 14:03:04 -0600 Subject: [PATCH 19/19] Remove trailing whitespace. --- Modules/_testinternalcapi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 685193194ff9b6..a33c8aa4f57023 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1574,7 +1574,7 @@ new_interpreter(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "|O:new_interpreter", &configobj)) { return NULL; } - + PyInterpreterConfig config; if (interp_config_from_object(configobj, &config) < 0) { return NULL;