From 8eb5adf33887ac566b0cc579866363f3d5a5f8d4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 14 Dec 2016 06:52:38 -0700 Subject: [PATCH 01/28] Add the _interpreters module to the stdlib. --- Doc/library/_interpreters.rst | 37 +++++++++++++++++++++++++++++ Lib/test/test__interpreters.py | 11 +++++++++ Modules/_interpretersmodule.c | 43 ++++++++++++++++++++++++++++++++++ setup.py | 3 +++ 4 files changed, 94 insertions(+) create mode 100644 Doc/library/_interpreters.rst create mode 100644 Lib/test/test__interpreters.py create mode 100644 Modules/_interpretersmodule.c diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst new file mode 100644 index 00000000000000..db0f2ef3942945 --- /dev/null +++ b/Doc/library/_interpreters.rst @@ -0,0 +1,37 @@ +:mod:`_interpreters` --- Low-level interpreters API +=================================================== + +.. module:: _interpreters + :synopsis: Low-level interpreters API. + +.. versionadded:: 3,7 + + :ref:`_sub-interpreter-support` + +threading + +-------------- + +This module provides low-level primitives for working with multiple +Python interpreters in the same process. + +.. XXX The :mod:`interpreters` module provides an easier to use and + higher-level API built on top of this module. + +This module is optional. It is provided by Python implementations which +support multiple interpreters. + +.. XXX For systems lacking the :mod:`_interpreters` module, the + :mod:`_dummy_interpreters` module is available. It duplicates this + module's interface and can be used as a drop-in replacement. + +It defines the following functions: + + +.. XXX TBD + + +**Caveats:** + +* ... + diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py new file mode 100644 index 00000000000000..157ef203b74907 --- /dev/null +++ b/Lib/test/test__interpreters.py @@ -0,0 +1,11 @@ +import unittest +from test import support +interpreters = support.import_module('_interpreters') + + +class InterpretersTests(unittest.TestCase): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c new file mode 100644 index 00000000000000..1103e1e37c48f1 --- /dev/null +++ b/Modules/_interpretersmodule.c @@ -0,0 +1,43 @@ + +/* interpreters module */ +/* low-level access to interpreter primitives */ + +#include "Python.h" + + +static PyMethodDef module_functions[] = { + {NULL, NULL} /* sentinel */ +}; + + +/* initialization function */ + +PyDoc_STRVAR(module_doc, +"This module provides primitive operations to manage Python interpreters.\n\ +The 'interpreters' module provides a more convenient interface."); + +static struct PyModuleDef interpretersmodule = { + PyModuleDef_HEAD_INIT, + "_interpreters", + module_doc, + -1, + module_functions, + NULL, + NULL, + NULL, + NULL +}; + + +PyMODINIT_FUNC +PyInit__interpreters(void) +{ + PyObject *module; + + module = PyModule_Create(&interpretersmodule); + if (module == NULL) + return NULL; + + + return module; +} diff --git a/setup.py b/setup.py index c22de17f953396..531e237b66aa1f 100644 --- a/setup.py +++ b/setup.py @@ -740,6 +740,9 @@ def detect_modules(self): ['_xxtestfuzz/_xxtestfuzz.c', '_xxtestfuzz/fuzzer.c']) ) + # Python interface to subinterpreter C-API. + exts.append(Extension('_interpreters', ['_interpretersmodule.c'])) + # # Here ends the simple stuff. From here on, modules need certain # libraries, are platform-specific, or present other surprises. From 457567f7dc0c2a86204466fbb56350fcf5c87ce0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 Dec 2016 15:28:53 -0700 Subject: [PATCH 02/28] Add create() and destroy(). --- Doc/library/_interpreters.rst | 13 ++- Lib/test/test__interpreters.py | 157 ++++++++++++++++++++++++++++++- Modules/_interpretersmodule.c | 167 ++++++++++++++++++++++++++++++++- 3 files changed, 334 insertions(+), 3 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index db0f2ef3942945..259b3ff67f13c3 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -28,7 +28,18 @@ support multiple interpreters. It defines the following functions: -.. XXX TBD +.. function:: create() + + Initialize a new Python interpreter and return its identifier. The + interpreter will be created in the current thread and will remain + idle until something is run in it. + + +.. function:: destroy(id) + + Finalize and destroy the identified interpreter. + +.. XXX must not be running? **Caveats:** diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 157ef203b74907..dd84db647bf1d5 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -1,10 +1,165 @@ +import threading import unittest + from test import support interpreters = support.import_module('_interpreters') class InterpretersTests(unittest.TestCase): - pass + + def setUp(self): + self.ids = [] + self.lock = threading.Lock() + + def tearDown(self): + for id in self.ids: + try: + interpreters.destroy(id) + except RuntimeError: + pass # already destroyed + + def _create(self): + id = interpreters.create() + self.ids.append(id) + return id + + def test_create_in_main(self): + id = interpreters.create() + self.ids.append(id) + + self.assertIn(id, interpreters._enumerate()) + + def test_create_unique_id(self): + seen = set() + for _ in range(100): + id = self._create() + interpreters.destroy(id) + seen.add(id) + + self.assertEqual(len(seen), 100) + + def test_create_in_thread(self): + id = None + def f(): + nonlocal id + id = interpreters.create() + self.ids.append(id) + self.lock.acquire() + self.lock.release() + + t = threading.Thread(target=f) + with self.lock: + t.start() + t.join() + self.assertIn(id, interpreters._enumerate()) + + @unittest.skip('waiting for run_string()') + def test_create_in_subinterpreter(self): + raise NotImplementedError + + @unittest.skip('waiting for run_string()') + def test_create_in_threaded_subinterpreter(self): + raise NotImplementedError + + def test_create_after_destroy_all(self): + before = set(interpreters._enumerate()) + # Create 3 subinterpreters. + ids = [] + for _ in range(3): + id = interpreters.create() + ids.append(id) + # Now destroy them. + for id in ids: + interpreters.destroy(id) + # Finally, create another. + id = interpreters.create() + self.ids.append(id) + self.assertEqual(set(interpreters._enumerate()), before | {id}) + + def test_create_after_destroy_some(self): + before = set(interpreters._enumerate()) + # Create 3 subinterpreters. + id1 = interpreters.create() + id2 = interpreters.create() + self.ids.append(id2) + id3 = interpreters.create() + # Now destroy 2 of them. + interpreters.destroy(id1) + interpreters.destroy(id3) + # Finally, create another. + id = interpreters.create() + self.ids.append(id) + self.assertEqual(set(interpreters._enumerate()), before | {id, id2}) + + def test_destroy_one(self): + id1 = self._create() + id2 = self._create() + id3 = self._create() + self.assertIn(id2, interpreters._enumerate()) + interpreters.destroy(id2) + self.assertNotIn(id2, interpreters._enumerate()) + self.assertIn(id1, interpreters._enumerate()) + self.assertIn(id3, interpreters._enumerate()) + + def test_destroy_all(self): + before = set(interpreters._enumerate()) + ids = set() + for _ in range(3): + id = self._create() + ids.add(id) + self.assertEqual(set(interpreters._enumerate()), before | ids) + for id in ids: + interpreters.destroy(id) + self.assertEqual(set(interpreters._enumerate()), before) + + def test_destroy_main(self): + main, = interpreters._enumerate() + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + def f(): + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_destroy_already_destroyed(self): + id = interpreters.create() + interpreters.destroy(id) + with self.assertRaises(RuntimeError): + interpreters.destroy(id) + + def test_destroy_does_not_exist(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(1_000_000) + + def test_destroy_bad_id(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(-1) + + @unittest.skip('waiting for run_string()') + def test_destroy_from_current(self): + raise NotImplementedError + + @unittest.skip('waiting for run_string()') + def test_destroy_from_sibling(self): + raise NotImplementedError + + def test_destroy_from_other_thread(self): + id = interpreters.create() + self.ids.append(id) + def f(): + interpreters.destroy(id) + + t = threading.Thread(target=f) + t.start() + t.join() + + @unittest.skip('waiting for run_string()') + def test_destroy_still_running(self): + raise NotImplementedError if __name__ == "__main__": diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 1103e1e37c48f1..97235034e722e7 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -5,8 +5,173 @@ #include "Python.h" +static PyObject * +_get_id(PyInterpreterState *interp) +{ + unsigned long id = PyInterpreterState_GetID(interp); + if (id == 0 && PyErr_Occurred() != NULL) + return NULL; + return PyLong_FromUnsignedLong(id); +} + +static PyInterpreterState * +_look_up(PyObject *requested_id) +{ + PyObject * id; + PyInterpreterState *interp; + + interp = PyInterpreterState_Head(); + while (interp != NULL) { + id = _get_id(interp); + if (id == NULL) + return NULL; + if (requested_id == id) + return interp; + interp = PyInterpreterState_Next(interp); + } + + PyErr_Format(PyExc_RuntimeError, + "unrecognized interpreter ID %R", requested_id); + return NULL; +} + +static PyObject * +_get_current(void) +{ + PyThreadState *tstate; + PyInterpreterState *interp; + + tstate = PyThreadState_Get(); + if (tstate == NULL) + return NULL; + interp = tstate->interp; + + // get ID + return _get_id(interp); +} + + +/* module level code ********************************************************/ + +// XXX track count? + +static PyObject * +interp_create(PyObject *self, PyObject *args) +{ + if (!PyArg_UnpackTuple(args, "create", 0, 0)) + return NULL; + + // Create and initialize the new interpreter. + PyThreadState *tstate, *save_tstate; + save_tstate = PyThreadState_Swap(NULL); + tstate = Py_NewInterpreter(); + PyThreadState_Swap(save_tstate); + if (tstate == NULL) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); + return NULL; + } + return _get_id(tstate->interp); +} + +PyDoc_STRVAR(create_doc, +"create() -> ID\n\ +\n\ +Create a new interpreter and return a unique generated ID."); + + +static PyObject * +interp_destroy(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "destroy", 1, 1, &id)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "ID must be an int"); + return NULL; + } + + // Ensure the ID is not the current interpreter. + PyObject *current = _get_current(); + if (current == NULL) + return NULL; + if (PyObject_RichCompareBool(id, current, Py_EQ) != 0) { + PyErr_SetString(PyExc_RuntimeError, + "cannot destroy the current interpreter"); + return NULL; + } + + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + + // Destroy the interpreter. + //PyInterpreterState_Delete(interp); + PyThreadState *tstate, *save_tstate; + tstate = PyInterpreterState_ThreadHead(interp); // XXX Is this the right one? + save_tstate = PyThreadState_Swap(tstate); + // XXX Stop current execution? + Py_EndInterpreter(tstate); // XXX Handle possible errors? + PyThreadState_Swap(save_tstate); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(destroy_doc, +"destroy(ID)\n\ +\n\ +Destroy the identified interpreter.\n\ +\n\ +Attempting to destroy the current interpreter results in a RuntimeError.\n\ +So does an unrecognized ID."); + + +static PyObject * +interp_enumerate(PyObject *self) +{ + PyObject *ids, *id; + PyInterpreterState *interp; + + // XXX Handle multiple main interpreters. + + ids = PyList_New(0); + if (ids == NULL) + return NULL; + + interp = PyInterpreterState_Head(); + while (interp != NULL) { + id = _get_id(interp); + if (id == NULL) + return NULL; + // insert at front of list + if (PyList_Insert(ids, 0, id) < 0) + return NULL; + + interp = PyInterpreterState_Next(interp); + } + + return ids; +} + +PyDoc_STRVAR(enumerate_doc, +"enumerate() -> [ID]\n\ +\n\ +Return a list containing the ID of every existing interpreter."); + + static PyMethodDef module_functions[] = { - {NULL, NULL} /* sentinel */ + {"create", (PyCFunction)interp_create, + METH_VARARGS, create_doc}, + {"destroy", (PyCFunction)interp_destroy, + METH_VARARGS, destroy_doc}, + + {"_enumerate", (PyCFunction)interp_enumerate, + METH_NOARGS, enumerate_doc}, + + {NULL, NULL} /* sentinel */ }; From e6af9f31212a0ae24112ccbb357a4088959d3dfb Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 31 Dec 2016 18:49:38 -0700 Subject: [PATCH 03/28] Finish nearly all the create/destroy tests. --- Lib/test/test__interpreters.py | 162 +++++++++++++++++++--------- Modules/_interpretersmodule.c | 186 +++++++++++++++++++++++++++------ 2 files changed, 265 insertions(+), 83 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index dd84db647bf1d5..c46ca0236b96d2 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -1,67 +1,108 @@ +import contextlib +import os import threading import unittest from test import support + interpreters = support.import_module('_interpreters') -class InterpretersTests(unittest.TestCase): +@contextlib.contextmanager +def _blocked(): + r, w = os.pipe() + wait_script = """if True: + import select + # Wait for a "done" signal. + select.select([{}], [], []) + + #import time + #time.sleep(1_000_000) + """.format(r) + try: + yield wait_script + finally: + os.write(w, b'') # release! + os.close(r) + os.close(w) + - def setUp(self): - self.ids = [] - self.lock = threading.Lock() +class TestBase(unittest.TestCase): def tearDown(self): - for id in self.ids: + for id in interpreters._enumerate(): + if id == 0: # main + continue try: interpreters.destroy(id) except RuntimeError: pass # already destroyed - def _create(self): - id = interpreters.create() - self.ids.append(id) - return id - def test_create_in_main(self): +class CreateTests(TestBase): + + def test_in_main(self): id = interpreters.create() - self.ids.append(id) self.assertIn(id, interpreters._enumerate()) - def test_create_unique_id(self): + def test_unique_id(self): seen = set() for _ in range(100): - id = self._create() + id = interpreters.create() interpreters.destroy(id) seen.add(id) self.assertEqual(len(seen), 100) - def test_create_in_thread(self): + def test_in_thread(self): + lock = threading.Lock() id = None def f(): nonlocal id id = interpreters.create() - self.ids.append(id) - self.lock.acquire() - self.lock.release() + lock.acquire() + lock.release() t = threading.Thread(target=f) - with self.lock: + with lock: t.start() t.join() self.assertIn(id, interpreters._enumerate()) - @unittest.skip('waiting for run_string()') - def test_create_in_subinterpreter(self): - raise NotImplementedError + def test_in_subinterpreter(self): + main, = interpreters._enumerate() + id = interpreters.create() + interpreters._run_string(id, """if True: + import _interpreters + id = _interpreters.create() + #_interpreters.create() + """) + + ids = interpreters._enumerate() + self.assertIn(id, ids) + self.assertIn(main, ids) + self.assertEqual(len(ids), 3) + + def test_in_threaded_subinterpreter(self): + main, = interpreters._enumerate() + id = interpreters.create() + def f(): + interpreters._run_string(id, """if True: + import _interpreters + _interpreters.create() + """) + + t = threading.Thread(target=f) + t.start() + t.join() - @unittest.skip('waiting for run_string()') - def test_create_in_threaded_subinterpreter(self): - raise NotImplementedError + ids = interpreters._enumerate() + self.assertIn(id, ids) + self.assertIn(main, ids) + self.assertEqual(len(ids), 3) - def test_create_after_destroy_all(self): + def test_after_destroy_all(self): before = set(interpreters._enumerate()) # Create 3 subinterpreters. ids = [] @@ -73,46 +114,46 @@ def test_create_after_destroy_all(self): interpreters.destroy(id) # Finally, create another. id = interpreters.create() - self.ids.append(id) self.assertEqual(set(interpreters._enumerate()), before | {id}) - def test_create_after_destroy_some(self): + def test_after_destroy_some(self): before = set(interpreters._enumerate()) # Create 3 subinterpreters. id1 = interpreters.create() id2 = interpreters.create() - self.ids.append(id2) id3 = interpreters.create() # Now destroy 2 of them. interpreters.destroy(id1) interpreters.destroy(id3) # Finally, create another. id = interpreters.create() - self.ids.append(id) self.assertEqual(set(interpreters._enumerate()), before | {id, id2}) - def test_destroy_one(self): - id1 = self._create() - id2 = self._create() - id3 = self._create() + +class DestroyTests(TestBase): + + def test_one(self): + id1 = interpreters.create() + id2 = interpreters.create() + id3 = interpreters.create() self.assertIn(id2, interpreters._enumerate()) interpreters.destroy(id2) self.assertNotIn(id2, interpreters._enumerate()) self.assertIn(id1, interpreters._enumerate()) self.assertIn(id3, interpreters._enumerate()) - def test_destroy_all(self): + def test_all(self): before = set(interpreters._enumerate()) ids = set() for _ in range(3): - id = self._create() + id = interpreters.create() ids.add(id) self.assertEqual(set(interpreters._enumerate()), before | ids) for id in ids: interpreters.destroy(id) self.assertEqual(set(interpreters._enumerate()), before) - def test_destroy_main(self): + def test_main(self): main, = interpreters._enumerate() with self.assertRaises(RuntimeError): interpreters.destroy(main) @@ -125,31 +166,40 @@ def f(): t.start() t.join() - def test_destroy_already_destroyed(self): + def test_already_destroyed(self): id = interpreters.create() interpreters.destroy(id) with self.assertRaises(RuntimeError): interpreters.destroy(id) - def test_destroy_does_not_exist(self): + def test_does_not_exist(self): with self.assertRaises(RuntimeError): interpreters.destroy(1_000_000) - def test_destroy_bad_id(self): + def test_bad_id(self): with self.assertRaises(RuntimeError): interpreters.destroy(-1) - @unittest.skip('waiting for run_string()') - def test_destroy_from_current(self): - raise NotImplementedError + def test_from_current(self): + id = interpreters.create() + with self.assertRaises(RuntimeError): + interpreters._run_string(id, """if True: + import _interpreters + _interpreters.destroy({}) + """.format(id)) - @unittest.skip('waiting for run_string()') - def test_destroy_from_sibling(self): - raise NotImplementedError + def test_from_sibling(self): + main, = interpreters._enumerate() + id1 = interpreters.create() + id2 = interpreters.create() + interpreters._run_string(id1, """if True: + import _interpreters + _interpreters.destroy({}) + """.format(id2)) + self.assertEqual(set(interpreters._enumerate()), {main, id1}) - def test_destroy_from_other_thread(self): + def test_from_other_thread(self): id = interpreters.create() - self.ids.append(id) def f(): interpreters.destroy(id) @@ -157,9 +207,21 @@ def f(): t.start() t.join() - @unittest.skip('waiting for run_string()') - def test_destroy_still_running(self): - raise NotImplementedError + @unittest.skip('not working yet') + def test_still_running(self): + main, = interpreters._enumerate() + id = interpreters.create() + def f(): + interpreters._run_string(id, wait_script) + + t = threading.Thread(target=f) + with _blocked() as wait_script: + t.start() + with self.assertRaises(RuntimeError): + interpreters.destroy(id) + + t.join() + self.assertEqual(set(interpreters._enumerate()), {main, id}) if __name__ == "__main__": diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 97235034e722e7..ce44df269bacd2 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -3,53 +3,131 @@ /* low-level access to interpreter primitives */ #include "Python.h" +#include "frameobject.h" -static PyObject * -_get_id(PyInterpreterState *interp) +static PyInterpreterState * +_get_current(void) { - unsigned long id = PyInterpreterState_GetID(interp); - if (id == 0 && PyErr_Occurred() != NULL) + PyThreadState *tstate; + + tstate = PyThreadState_Get(); + if (tstate == NULL) return NULL; - return PyLong_FromUnsignedLong(id); + return tstate->interp; } static PyInterpreterState * -_look_up(PyObject *requested_id) +_look_up_int64(PY_INT64_T requested_id) { - PyObject * id; - PyInterpreterState *interp; + if (requested_id < 0) + goto error; - interp = PyInterpreterState_Head(); + PyInterpreterState *interp = PyInterpreterState_Head(); while (interp != NULL) { - id = _get_id(interp); - if (id == NULL) + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) return NULL; if (requested_id == id) return interp; interp = PyInterpreterState_Next(interp); } +error: PyErr_Format(PyExc_RuntimeError, - "unrecognized interpreter ID %R", requested_id); + "unrecognized interpreter ID %lld", requested_id); return NULL; } -static PyObject * -_get_current(void) +static PyInterpreterState * +_look_up(PyObject *requested_id) { - PyThreadState *tstate; - PyInterpreterState *interp; + long long id = PyLong_AsLongLong(requested_id); + if (id == -1 && PyErr_Occurred() != NULL) + return NULL; + // XXX Fail if larger than INT64_MAX? + return _look_up_int64(id); +} - tstate = PyThreadState_Get(); - if (tstate == NULL) +static PyObject * +_get_id(PyInterpreterState *interp) +{ + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) return NULL; - interp = tstate->interp; + return PyLong_FromLongLong(id); +} - // get ID - return _get_id(interp); +static int +_is_running(PyInterpreterState *interp) +{ + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + if (PyThreadState_Next(tstate) != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "interpreter has more than one thread"); + return -1; + } + PyFrameObject *frame = _PyThreadState_GetFrame(tstate); + if (frame == NULL) { + if (PyErr_Occurred() != NULL) + return -1; + return 0; + } + return (int)(frame->f_executing); } +static int +_ensure_not_running(PyInterpreterState *interp) +{ + int is_running = _is_running(interp); + if (is_running < 0) + return -1; + if (is_running) { + PyErr_Format(PyExc_RuntimeError, "interpreter already running"); + return -1; + } + return 0; +} + +static int +_run_string(PyInterpreterState *interp, const char *codestr) +{ + PyObject *result = NULL; + PyObject *exc = NULL, *value = NULL, *tb = NULL; + + if (_ensure_not_running(interp) < 0) + return -1; + + // Switch to interpreter. + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + PyThreadState *save_tstate = PyThreadState_Swap(tstate); + + // Run the string (see PyRun_SimpleStringFlags). + // XXX How to handle sys.exit()? + PyObject *m = PyImport_AddModule("__main__"); + if (m == NULL) { + PyErr_Fetch(&exc, &value, &tb); + goto done; + } + PyObject *d = PyModule_GetDict(m); + result = PyRun_StringFlags(codestr, Py_file_input, d, d, NULL); + if (result == NULL) { + // Get the exception from the subinterpreter. + PyErr_Fetch(&exc, &value, &tb); + goto done; + } + Py_DECREF(result); // We throw away the result. + +done: + // Switch back. + if (save_tstate != NULL) + PyThreadState_Swap(save_tstate); + + // Propagate any exception out to the caller. + PyErr_Restore(exc, value, tb); + + return (result == NULL) ? -1 : 0; +} /* module level code ********************************************************/ @@ -93,19 +171,25 @@ interp_destroy(PyObject *self, PyObject *args) return NULL; } - // Ensure the ID is not the current interpreter. - PyObject *current = _get_current(); + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + + // Ensure we don't try to destroy the current interpreter. + PyInterpreterState *current = _get_current(); if (current == NULL) return NULL; - if (PyObject_RichCompareBool(id, current, Py_EQ) != 0) { + if (interp == current) { PyErr_SetString(PyExc_RuntimeError, "cannot destroy the current interpreter"); return NULL; } - // Look up the interpreter. - PyInterpreterState *interp = _look_up(id); - if (interp == NULL) + // Ensure the interpreter isn't running. + /* XXX We *could* support destroying a running interpreter but + aren't going to worry about it for now. */ + if (_ensure_not_running(interp) < 0) return NULL; // Destroy the interpreter. @@ -156,11 +240,44 @@ interp_enumerate(PyObject *self) return ids; } -PyDoc_STRVAR(enumerate_doc, -"enumerate() -> [ID]\n\ -\n\ -Return a list containing the ID of every existing interpreter."); +static PyObject * +interp_run_string(PyObject *self, PyObject *args) +{ + PyObject *id, *code; + if (!PyArg_UnpackTuple(args, "run_string", 2, 2, &id, &code)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int"); + return NULL; + } + if (!PyUnicode_Check(code)) { + PyErr_SetString(PyExc_TypeError, + "second arg (code) must be a string"); + return NULL; + } + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + + // Extract code. + Py_ssize_t size; + const char *codestr = PyUnicode_AsUTF8AndSize(code, &size); + if (codestr == NULL) + return NULL; + if (strlen(codestr) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, + "source code string cannot contain null bytes"); + return NULL; + } + + // Run the code in the interpreter. + if (_run_string(interp, codestr) < 0) + return NULL; + else + Py_RETURN_NONE; +} static PyMethodDef module_functions[] = { {"create", (PyCFunction)interp_create, @@ -168,8 +285,11 @@ static PyMethodDef module_functions[] = { {"destroy", (PyCFunction)interp_destroy, METH_VARARGS, destroy_doc}, - {"_enumerate", (PyCFunction)interp_enumerate, - METH_NOARGS, enumerate_doc}, + {"_enumerate", (PyCFunction)interp_enumerate, + METH_NOARGS, NULL}, + + {"_run_string", (PyCFunction)interp_run_string, + METH_VARARGS, NULL}, {NULL, NULL} /* sentinel */ }; From 335967e0e36d60047992f0ef757a8c3c42d0d71f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 Dec 2016 15:32:18 -0700 Subject: [PATCH 04/28] Add run_string(). --- Doc/library/_interpreters.rst | 18 +++- Lib/test/test__interpreters.py | 190 ++++++++++++++++++++++++++++++--- Modules/_interpretersmodule.c | 11 +- 3 files changed, 203 insertions(+), 16 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 259b3ff67f13c3..8d05622f5af106 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -27,7 +27,6 @@ support multiple interpreters. It defines the following functions: - .. function:: create() Initialize a new Python interpreter and return its identifier. The @@ -42,6 +41,23 @@ It defines the following functions: .. XXX must not be running? +.. function:: run_string(id, command) + + A wrapper around :c:func:`PyRun_SimpleString` which runs the provided + Python program using the identified interpreter. Providing an + invalid or unknown ID results in a RuntimeError, likewise if the main + interpreter or any other running interpreter is used. + + Any value returned from the code is thrown away, similar to what + threads do. If the code results in an exception then that exception + is raised in the thread in which run_string() was called, similar to + how :func:`exec` works. This aligns with how interpreters are not + inherently threaded. + +.. XXX must not be running already? +.. XXX sys.exit() (and SystemExit) is swallowed? + + **Caveats:** * ... diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index c46ca0236b96d2..f6392ae7abaeab 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -1,5 +1,9 @@ import contextlib import os +import os.path +import shutil +import tempfile +from textwrap import dedent import threading import unittest @@ -11,14 +15,14 @@ @contextlib.contextmanager def _blocked(): r, w = os.pipe() - wait_script = """if True: + wait_script = dedent(""" import select # Wait for a "done" signal. select.select([{}], [], []) #import time #time.sleep(1_000_000) - """.format(r) + """).format(r) try: yield wait_script finally: @@ -73,11 +77,10 @@ def f(): def test_in_subinterpreter(self): main, = interpreters._enumerate() id = interpreters.create() - interpreters._run_string(id, """if True: + interpreters.run_string(id, dedent(""" import _interpreters id = _interpreters.create() - #_interpreters.create() - """) + """)) ids = interpreters._enumerate() self.assertIn(id, ids) @@ -88,10 +91,10 @@ def test_in_threaded_subinterpreter(self): main, = interpreters._enumerate() id = interpreters.create() def f(): - interpreters._run_string(id, """if True: + interpreters.run_string(id, dedent(""" import _interpreters _interpreters.create() - """) + """)) t = threading.Thread(target=f) t.start() @@ -102,6 +105,7 @@ def f(): self.assertIn(main, ids) self.assertEqual(len(ids), 3) + def test_after_destroy_all(self): before = set(interpreters._enumerate()) # Create 3 subinterpreters. @@ -183,19 +187,19 @@ def test_bad_id(self): def test_from_current(self): id = interpreters.create() with self.assertRaises(RuntimeError): - interpreters._run_string(id, """if True: + interpreters.run_string(id, dedent(""" import _interpreters _interpreters.destroy({}) - """.format(id)) + """).format(id)) def test_from_sibling(self): main, = interpreters._enumerate() id1 = interpreters.create() id2 = interpreters.create() - interpreters._run_string(id1, """if True: + interpreters.run_string(id1, dedent(""" import _interpreters _interpreters.destroy({}) - """.format(id2)) + """).format(id2)) self.assertEqual(set(interpreters._enumerate()), {main, id1}) def test_from_other_thread(self): @@ -212,7 +216,7 @@ def test_still_running(self): main, = interpreters._enumerate() id = interpreters.create() def f(): - interpreters._run_string(id, wait_script) + interpreters.run_string(id, wait_script) t = threading.Thread(target=f) with _blocked() as wait_script: @@ -224,5 +228,165 @@ def f(): self.assertEqual(set(interpreters._enumerate()), {main, id}) -if __name__ == "__main__": +class RunStringTests(TestBase): + + SCRIPT = dedent(""" + with open('{}', 'w') as out: + out.write('{}') + """) + FILENAME = 'spam' + + def setUp(self): + self.id = interpreters.create() + self.dirname = None + self.filename = None + + def tearDown(self): + if self.dirname is not None: + shutil.rmtree(self.dirname) + super().tearDown() + + def _resolve_filename(self, name=None): + if name is None: + name = self.FILENAME + if self.dirname is None: + self.dirname = tempfile.mkdtemp() + return os.path.join(self.dirname, name) + + def _empty_file(self): + self.filename = self._resolve_filename() + support.create_empty_file(self.filename) + return self.filename + + def assert_file_contains(self, expected, filename=None): + if filename is None: + filename = self.filename + self.assertIsNot(filename, None) + with open(filename) as out: + content = out.read() + self.assertEqual(content, expected) + + def test_success(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = self.SCRIPT.format(filename, expected) + interpreters.run_string(self.id, script) + + self.assert_file_contains(expected) + + def test_in_thread(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = self.SCRIPT.format(filename, expected) + def f(): + interpreters.run_string(self.id, script) + + t = threading.Thread(target=f) + t.start() + t.join() + + self.assert_file_contains(expected) + + def test_create_thread(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = dedent(""" + import threading + def f(): + with open('{}', 'w') as out: + out.write('{}') + + t = threading.Thread(target=f) + t.start() + t.join() + """).format(filename, expected) + interpreters.run_string(self.id, script) + + self.assert_file_contains(expected) + + @unittest.skip('not working yet') + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + def test_fork(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = dedent(""" + import os + import sys + pid = os.fork() + if pid == 0: + with open('{}', 'w') as out: + out.write('{}') + sys.exit(0) + """).format(filename, expected) + interpreters.run_string(self.id, script) + + self.assert_file_contains(expected) + + @unittest.skip('not working yet') + def test_already_running(self): + def f(): + interpreters.run_string(self.id, wait_script) + + t = threading.Thread(target=f) + with _blocked() as wait_script: + t.start() + with self.assertRaises(RuntimeError): + interpreters.run_string(self.id, 'print("spam")') + t.join() + + def test_does_not_exist(self): + id = 0 + while id in interpreters._enumerate(): + id += 1 + with self.assertRaises(RuntimeError): + interpreters.run_string(id, 'print("spam")') + + def test_error_id(self): + with self.assertRaises(RuntimeError): + interpreters.run_string(-1, 'print("spam")') + + def test_bad_id(self): + with self.assertRaises(TypeError): + interpreters.run_string('spam', 'print("spam")') + + def test_bad_code(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, 10) + + def test_bytes_for_code(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, b'print("spam")') + + def test_invalid_syntax(self): + with self.assertRaises(SyntaxError): + # missing close paren + interpreters.run_string(self.id, 'print("spam"') + + def test_failure(self): + with self.assertRaises(Exception) as caught: + interpreters.run_string(self.id, 'raise Exception("spam")') + self.assertEqual(str(caught.exception), 'spam') + + def test_sys_exit(self): + with self.assertRaises(SystemExit) as cm: + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit() + """)) + self.assertIsNone(cm.exception.code) + + with self.assertRaises(SystemExit) as cm: + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit(42) + """)) + self.assertEqual(cm.exception.code, 42) + + def test_SystemError(self): + with self.assertRaises(SystemExit) as cm: + interpreters.run_string(self.id, 'raise SystemExit(42)') + self.assertEqual(cm.exception.code, 42) + + +if __name__ == '__main__': unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index ce44df269bacd2..a46610e5a10899 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -279,6 +279,13 @@ interp_run_string(PyObject *self, PyObject *args) Py_RETURN_NONE; } +PyDoc_STRVAR(run_string_doc, +"run_string(ID, sourcetext) -> run_id\n\ +\n\ +Execute the provided string in the identified interpreter.\n\ +See PyRun_SimpleStrings."); + + static PyMethodDef module_functions[] = { {"create", (PyCFunction)interp_create, METH_VARARGS, create_doc}, @@ -288,8 +295,8 @@ static PyMethodDef module_functions[] = { {"_enumerate", (PyCFunction)interp_enumerate, METH_NOARGS, NULL}, - {"_run_string", (PyCFunction)interp_run_string, - METH_VARARGS, NULL}, + {"run_string", (PyCFunction)interp_run_string, + METH_VARARGS, run_string_doc}, {NULL, NULL} /* sentinel */ }; From d2df97381f25188fb7c735b46720d944af266921 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Jan 2017 15:42:08 -0700 Subject: [PATCH 05/28] Get tricky tests working. --- Lib/test/test__interpreters.py | 53 +++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index f6392ae7abaeab..a23246ea2a3f3a 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -13,22 +13,18 @@ @contextlib.contextmanager -def _blocked(): - r, w = os.pipe() +def _blocked(dirname): + filename = os.path.join(dirname, '.lock') wait_script = dedent(""" - import select - # Wait for a "done" signal. - select.select([{}], [], []) - - #import time - #time.sleep(1_000_000) - """).format(r) + import os.path + import time + while not os.path.exists('{}'): + time.sleep(0.1) + """).format(filename) try: yield wait_script finally: - os.write(w, b'') # release! - os.close(r) - os.close(w) + support.create_empty_file(filename) class TestBase(unittest.TestCase): @@ -211,15 +207,15 @@ def f(): t.start() t.join() - @unittest.skip('not working yet') def test_still_running(self): main, = interpreters._enumerate() id = interpreters.create() def f(): interpreters.run_string(id, wait_script) + dirname = tempfile.mkdtemp() t = threading.Thread(target=f) - with _blocked() as wait_script: + with _blocked(dirname) as wait_script: t.start() with self.assertRaises(RuntimeError): interpreters.destroy(id) @@ -243,7 +239,10 @@ def setUp(self): def tearDown(self): if self.dirname is not None: - shutil.rmtree(self.dirname) + try: + shutil.rmtree(self.dirname) + except FileNotFoundError: + pass # already deleted super().tearDown() def _resolve_filename(self, name=None): @@ -304,31 +303,39 @@ def f(): self.assert_file_contains(expected) - @unittest.skip('not working yet') @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") def test_fork(self): filename = self._empty_file() expected = 'spam spam spam spam spam' script = dedent(""" import os - import sys + r, w = os.pipe() pid = os.fork() - if pid == 0: - with open('{}', 'w') as out: + if pid == 0: # child + import sys + filename = '{}' + with open(filename, 'w') as out: out.write('{}') - sys.exit(0) + os.write(w, b'done!') + else: + import select + try: + select.select([r], [], []) + finally: + os.close(r) + os.close(w) """).format(filename, expected) + # XXX Kill the child process in a unittest-friendly way. interpreters.run_string(self.id, script) - self.assert_file_contains(expected) - @unittest.skip('not working yet') def test_already_running(self): def f(): interpreters.run_string(self.id, wait_script) t = threading.Thread(target=f) - with _blocked() as wait_script: + dirname = tempfile.mkdtemp() + with _blocked(dirname) as wait_script: t.start() with self.assertRaises(RuntimeError): interpreters.run_string(self.id, 'print("spam")') From b70a496e509bced08eb22a0931418ecb66e735e0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Jan 2017 15:46:07 -0700 Subject: [PATCH 06/28] Add a test for a still running interpreter when main exits. --- Lib/test/test__interpreters.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index a23246ea2a3f3a..24724b662dd400 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -8,10 +8,22 @@ import unittest from test import support +from test.support import script_helper interpreters = support.import_module('_interpreters') +SCRIPT_THREADED_INTERP = """\ +import threading +import _interpreters +def f(): + _interpreters.run_string(id, {!r}) + +t = threading.Thread(target=f) +t.start() +""" + + @contextlib.contextmanager def _blocked(dirname): filename = os.path.join(dirname, '.lock') @@ -27,6 +39,27 @@ def _blocked(dirname): support.create_empty_file(filename) +class InterpreterTests(unittest.TestCase): + + def setUp(self): + self.dirname = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.dirname) + + def test_still_running_at_exit(self): + script = SCRIPT_THREADED_INTERP.format("""if True: + import time + # Give plenty of time for the main interpreter to finish. + time.sleep(1_000_000) + """) + filename = script_helper.make_script(self.dirname, 'interp', script) + proc = script_helper.spawn_python(filename) + retcode = proc.wait() + + self.assertEqual(retcode, 0) + + class TestBase(unittest.TestCase): def tearDown(self): From 6f2d28fd43e60817f34ca950732ad7385c81f5be Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jan 2017 17:48:34 -0700 Subject: [PATCH 07/28] Add run_string_unrestricted(). --- Doc/library/_interpreters.rst | 12 +++- Lib/test/test__interpreters.py | 101 ++++++++++++++++++++++++++------- Modules/_interpretersmodule.c | 95 +++++++++++++++++++++++++------ 3 files changed, 168 insertions(+), 40 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 8d05622f5af106..a4f86e38496e5b 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -38,8 +38,6 @@ It defines the following functions: Finalize and destroy the identified interpreter. -.. XXX must not be running? - .. function:: run_string(id, command) @@ -54,10 +52,18 @@ It defines the following functions: how :func:`exec` works. This aligns with how interpreters are not inherently threaded. -.. XXX must not be running already? .. XXX sys.exit() (and SystemExit) is swallowed? +.. function:: run_string_unrestricted(id, command, ns=None) + + Like :c:func:`run_string` but returns the dict in which the code + was executed. It also supports providing a namespace that gets + merged into the execution namespace before execution. Note that + this allows objects to leak between interpreters, which may not + be desirable. + + **Caveats:** * ... diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 24724b662dd400..edd279c16c4935 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -105,34 +105,33 @@ def f(): def test_in_subinterpreter(self): main, = interpreters._enumerate() - id = interpreters.create() - interpreters.run_string(id, dedent(""" + id1 = interpreters.create() + ns = interpreters.run_string_unrestricted(id1, dedent(""" import _interpreters id = _interpreters.create() """)) + id2 = ns['id'] - ids = interpreters._enumerate() - self.assertIn(id, ids) - self.assertIn(main, ids) - self.assertEqual(len(ids), 3) + self.assertEqual(set(interpreters._enumerate()), {main, id1, id2}) def test_in_threaded_subinterpreter(self): main, = interpreters._enumerate() - id = interpreters.create() + id1 = interpreters.create() + ns = None + script = dedent(""" + import _interpreters + id = _interpreters.create() + """) def f(): - interpreters.run_string(id, dedent(""" - import _interpreters - _interpreters.create() - """)) + nonlocal ns + ns = interpreters.run_string_unrestricted(id1, script) t = threading.Thread(target=f) t.start() t.join() + id2 = ns['id'] - ids = interpreters._enumerate() - self.assertIn(id, ids) - self.assertIn(main, ids) - self.assertEqual(len(ids), 3) + self.assertEqual(set(interpreters._enumerate()), {main, id1, id2}) def test_after_destroy_all(self): @@ -214,21 +213,27 @@ def test_bad_id(self): interpreters.destroy(-1) def test_from_current(self): + main, = interpreters._enumerate() id = interpreters.create() + script = dedent(""" + import _interpreters + _interpreters.destroy({}) + """).format(id) + with self.assertRaises(RuntimeError): - interpreters.run_string(id, dedent(""" - import _interpreters - _interpreters.destroy({}) - """).format(id)) + interpreters.run_string(id, script) + self.assertEqual(set(interpreters._enumerate()), {main, id}) def test_from_sibling(self): main, = interpreters._enumerate() id1 = interpreters.create() id2 = interpreters.create() - interpreters.run_string(id1, dedent(""" + script = dedent(""" import _interpreters _interpreters.destroy({}) - """).format(id2)) + """).format(id2) + interpreters.run_string(id1, script) + self.assertEqual(set(interpreters._enumerate()), {main, id1}) def test_from_other_thread(self): @@ -241,6 +246,8 @@ def f(): t.join() def test_still_running(self): + # XXX Rewrite this test without files by using + # run_string_unrestricted(). main, = interpreters._enumerate() id = interpreters.create() def f(): @@ -428,5 +435,57 @@ def test_SystemError(self): self.assertEqual(cm.exception.code, 42) +class RunStringUnrestrictedTests(TestBase): + + def setUp(self): + self.id = interpreters.create() + + def test_without_ns(self): + script = dedent(""" + spam = 42 + """) + ns = interpreters.run_string_unrestricted(self.id, script) + + self.assertEqual(ns['spam'], 42) + + def test_with_ns(self): + updates = {'spam': 'ham', 'eggs': -1} + script = dedent(""" + spam = 42 + result = spam + eggs + """) + ns = interpreters.run_string_unrestricted(self.id, script, updates) + + self.assertEqual(ns['spam'], 42) + self.assertEqual(ns['eggs'], -1) + self.assertEqual(ns['result'], 41) + + def test_ns_does_not_overwrite(self): + updates = {'__name__': 'not __main__'} + script = dedent(""" + spam = 42 + """) + ns = interpreters.run_string_unrestricted(self.id, script, updates) + + self.assertEqual(ns['__name__'], '__main__') + + def test_return_execution_namespace(self): + script = dedent(""" + spam = 42 + """) + ns = interpreters.run_string_unrestricted(self.id, script) + + ns.pop('__builtins__') + ns.pop('__loader__') + self.assertEqual(ns, { + '__name__': '__main__', + '__annotations__': {}, + '__doc__': None, + '__package__': None, + '__spec__': None, + 'spam': 42, + }) + + if __name__ == '__main__': unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index a46610e5a10899..47e4c3f1dcee40 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -89,14 +89,11 @@ _ensure_not_running(PyInterpreterState *interp) return 0; } -static int -_run_string(PyInterpreterState *interp, const char *codestr) +static PyObject * +_run_string(PyInterpreterState *interp, const char *codestr, PyObject *updates) { - PyObject *result = NULL; - PyObject *exc = NULL, *value = NULL, *tb = NULL; - if (_ensure_not_running(interp) < 0) - return -1; + return NULL; // Switch to interpreter. PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); @@ -104,19 +101,32 @@ _run_string(PyInterpreterState *interp, const char *codestr) // Run the string (see PyRun_SimpleStringFlags). // XXX How to handle sys.exit()? + PyObject *exc = NULL, *value = NULL, *tb = NULL; + PyObject *ns = NULL; + // XXX Force a fresh __main__ module? PyObject *m = PyImport_AddModule("__main__"); if (m == NULL) { PyErr_Fetch(&exc, &value, &tb); goto done; } - PyObject *d = PyModule_GetDict(m); - result = PyRun_StringFlags(codestr, Py_file_input, d, d, NULL); + ns = PyModule_GetDict(m); + if (ns == NULL) { + PyErr_Fetch(&exc, &value, &tb); + goto done; + } + if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) { + PyErr_Fetch(&exc, &value, &tb); + goto done; + } + PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); if (result == NULL) { + ns = NULL; // Get the exception from the subinterpreter. PyErr_Fetch(&exc, &value, &tb); goto done; } Py_DECREF(result); // We throw away the result. + Py_INCREF(ns); // It is a borrowed reference. done: // Switch back. @@ -126,7 +136,7 @@ _run_string(PyInterpreterState *interp, const char *codestr) // Propagate any exception out to the caller. PyErr_Restore(exc, value, tb); - return (result == NULL) ? -1 : 0; + return ns; } /* module level code ********************************************************/ @@ -273,32 +283,85 @@ interp_run_string(PyObject *self, PyObject *args) } // Run the code in the interpreter. - if (_run_string(interp, codestr) < 0) + PyObject *ns = _run_string(interp, codestr, NULL); + if (ns == NULL) return NULL; else Py_RETURN_NONE; } PyDoc_STRVAR(run_string_doc, -"run_string(ID, sourcetext) -> run_id\n\ +"run_string(ID, sourcetext)\n\ \n\ Execute the provided string in the identified interpreter.\n\ +\n\ +See PyRun_SimpleStrings."); + + +static PyObject * +interp_run_string_unrestricted(PyObject *self, PyObject *args) +{ + PyObject *id, *code, *ns = NULL; + if (!PyArg_UnpackTuple(args, "run_string_unrestricted", 2, 3, + &id, &code, &ns)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int"); + return NULL; + } + if (!PyUnicode_Check(code)) { + PyErr_SetString(PyExc_TypeError, + "second arg (code) must be a string"); + return NULL; + } + if (ns == Py_None) + ns = NULL; + + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + + // Extract code. + Py_ssize_t size; + const char *codestr = PyUnicode_AsUTF8AndSize(code, &size); + if (codestr == NULL) + return NULL; + if (strlen(codestr) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, + "source code string cannot contain null bytes"); + return NULL; + } + + // Run the code in the interpreter. + return _run_string(interp, codestr, ns); +} + +PyDoc_STRVAR(run_string_unrestricted_doc, +"run_string_unrestricted(ID, sourcetext, ns=None) -> main module ns\n\ +\n\ +Execute the provided string in the identified interpreter. Return the\n\ +dict in which the code executed. If the ns arg is provided then it is\n\ +merged into the execution namespace before the code is executed.\n\ +\n\ See PyRun_SimpleStrings."); static PyMethodDef module_functions[] = { - {"create", (PyCFunction)interp_create, + {"create", (PyCFunction)interp_create, METH_VARARGS, create_doc}, - {"destroy", (PyCFunction)interp_destroy, + {"destroy", (PyCFunction)interp_destroy, METH_VARARGS, destroy_doc}, - {"_enumerate", (PyCFunction)interp_enumerate, + {"_enumerate", (PyCFunction)interp_enumerate, METH_NOARGS, NULL}, - {"run_string", (PyCFunction)interp_run_string, + {"run_string", (PyCFunction)interp_run_string, METH_VARARGS, run_string_doc}, + {"run_string_unrestricted", (PyCFunction)interp_run_string_unrestricted, + METH_VARARGS, run_string_unrestricted_doc}, - {NULL, NULL} /* sentinel */ + {NULL, NULL} /* sentinel */ }; From 80a931b42a67795528315eb2eb8e54ca2b941240 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jan 2017 18:17:18 -0700 Subject: [PATCH 08/28] Exit out of the child process. --- Lib/test/test__interpreters.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index edd279c16c4935..1402872e3412d6 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -357,6 +357,9 @@ def test_fork(self): with open(filename, 'w') as out: out.write('{}') os.write(w, b'done!') + + # Kill the unittest runner in the child process. + os._exit(1) else: import select try: @@ -365,7 +368,6 @@ def test_fork(self): os.close(r) os.close(w) """).format(filename, expected) - # XXX Kill the child process in a unittest-friendly way. interpreters.run_string(self.id, script) self.assert_file_contains(expected) From 6aa7d9cf1878aa3b18b60a6f20169707755cc6e9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jan 2017 18:45:52 -0700 Subject: [PATCH 09/28] Resolve several TODOs. --- Doc/library/_interpreters.rst | 6 +++--- Modules/_interpretersmodule.c | 12 +++--------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index a4f86e38496e5b..e2d0191ac71703 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -50,9 +50,9 @@ It defines the following functions: threads do. If the code results in an exception then that exception is raised in the thread in which run_string() was called, similar to how :func:`exec` works. This aligns with how interpreters are not - inherently threaded. - -.. XXX sys.exit() (and SystemExit) is swallowed? + inherently threaded. Note that SystemExit (as raised by sys.exit()) + is not treated any differently and will result in the process ending + if not caught explicitly. .. function:: run_string_unrestricted(id, command, ns=None) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 47e4c3f1dcee40..a8d5b5391a1a64 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -45,7 +45,7 @@ _look_up(PyObject *requested_id) long long id = PyLong_AsLongLong(requested_id); if (id == -1 && PyErr_Occurred() != NULL) return NULL; - // XXX Fail if larger than INT64_MAX? + assert(id <= INT64_MAX); return _look_up_int64(id); } @@ -100,7 +100,6 @@ _run_string(PyInterpreterState *interp, const char *codestr, PyObject *updates) PyThreadState *save_tstate = PyThreadState_Swap(tstate); // Run the string (see PyRun_SimpleStringFlags). - // XXX How to handle sys.exit()? PyObject *exc = NULL, *value = NULL, *tb = NULL; PyObject *ns = NULL; // XXX Force a fresh __main__ module? @@ -141,8 +140,6 @@ _run_string(PyInterpreterState *interp, const char *codestr, PyObject *updates) /* module level code ********************************************************/ -// XXX track count? - static PyObject * interp_create(PyObject *self, PyObject *args) { @@ -205,10 +202,9 @@ interp_destroy(PyObject *self, PyObject *args) // Destroy the interpreter. //PyInterpreterState_Delete(interp); PyThreadState *tstate, *save_tstate; - tstate = PyInterpreterState_ThreadHead(interp); // XXX Is this the right one? + tstate = PyInterpreterState_ThreadHead(interp); save_tstate = PyThreadState_Swap(tstate); - // XXX Stop current execution? - Py_EndInterpreter(tstate); // XXX Handle possible errors? + Py_EndInterpreter(tstate); PyThreadState_Swap(save_tstate); Py_RETURN_NONE; @@ -229,8 +225,6 @@ interp_enumerate(PyObject *self) PyObject *ids, *id; PyInterpreterState *interp; - // XXX Handle multiple main interpreters. - ids = PyList_New(0); if (ids == NULL) return NULL; From 083e13c01f7c9a840cb8dd5a9268a41aa622f4b0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jan 2017 19:17:53 -0700 Subject: [PATCH 10/28] Set up the execution namespace before switching threads. --- Modules/_interpretersmodule.c | 66 ++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index a8d5b5391a1a64..6366c7ec5e7611 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -89,45 +89,23 @@ _ensure_not_running(PyInterpreterState *interp) return 0; } -static PyObject * -_run_string(PyInterpreterState *interp, const char *codestr, PyObject *updates) +static int +_run_string(PyInterpreterState *interp, const char *codestr, PyObject *ns) { - if (_ensure_not_running(interp) < 0) - return NULL; - // Switch to interpreter. PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); PyThreadState *save_tstate = PyThreadState_Swap(tstate); // Run the string (see PyRun_SimpleStringFlags). PyObject *exc = NULL, *value = NULL, *tb = NULL; - PyObject *ns = NULL; - // XXX Force a fresh __main__ module? - PyObject *m = PyImport_AddModule("__main__"); - if (m == NULL) { - PyErr_Fetch(&exc, &value, &tb); - goto done; - } - ns = PyModule_GetDict(m); - if (ns == NULL) { - PyErr_Fetch(&exc, &value, &tb); - goto done; - } - if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) { - PyErr_Fetch(&exc, &value, &tb); - goto done; - } PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); if (result == NULL) { - ns = NULL; // Get the exception from the subinterpreter. PyErr_Fetch(&exc, &value, &tb); - goto done; + } else { + Py_DECREF(result); // We throw away the result. } - Py_DECREF(result); // We throw away the result. - Py_INCREF(ns); // It is a borrowed reference. -done: // Switch back. if (save_tstate != NULL) PyThreadState_Swap(save_tstate); @@ -135,6 +113,38 @@ _run_string(PyInterpreterState *interp, const char *codestr, PyObject *updates) // Propagate any exception out to the caller. PyErr_Restore(exc, value, tb); + return result == NULL ? -1 : 0; +} + +static PyObject * +_run_string_in_main(PyInterpreterState *interp, const char *codestr, + PyObject *updates) +{ + if (_ensure_not_running(interp) < 0) + return NULL; + + // Get the namespace in which to execute. + // XXX Force a fresh __main__ module? + PyObject *m = PyMapping_GetItemString(interp->modules, "__main__"); + if (m == NULL) + return NULL; + PyObject *orig = PyModule_GetDict(m); // borrowed + Py_DECREF(m); + if (orig == NULL) + return NULL; + PyObject *ns = PyDict_Copy(orig); + if (ns == NULL) + return NULL; + if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) { + Py_DECREF(ns); + return NULL; + } + + if (_run_string(interp, codestr, ns) < 0) { + Py_DECREF(ns); + return NULL; + } + return ns; } @@ -277,7 +287,7 @@ interp_run_string(PyObject *self, PyObject *args) } // Run the code in the interpreter. - PyObject *ns = _run_string(interp, codestr, NULL); + PyObject *ns = _run_string_in_main(interp, codestr, NULL); if (ns == NULL) return NULL; else @@ -328,7 +338,7 @@ interp_run_string_unrestricted(PyObject *self, PyObject *args) } // Run the code in the interpreter. - return _run_string(interp, codestr, ns); + return _run_string_in_main(interp, codestr, ns); } PyDoc_STRVAR(run_string_unrestricted_doc, From ce081a7d49184cad516f80dfd0eaaf34548a1073 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jan 2017 23:42:29 -0700 Subject: [PATCH 11/28] Run in a copy of __main__. --- Lib/test/test__interpreters.py | 11 +++++- Modules/_interpretersmodule.c | 70 +++++++++++++++++++++++++++------- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 1402872e3412d6..3012f4e59ea83d 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -300,7 +300,7 @@ def _empty_file(self): def assert_file_contains(self, expected, filename=None): if filename is None: filename = self.filename - self.assertIsNot(filename, None) + self.assertIsNotNone(filename) with open(filename) as out: content = out.read() self.assertEqual(content, expected) @@ -471,6 +471,15 @@ def test_ns_does_not_overwrite(self): self.assertEqual(ns['__name__'], '__main__') + def test_main_not_shared(self): + ns1 = interpreters.run_string_unrestricted(self.id, 'spam = True') + ns2 = interpreters.run_string_unrestricted(self.id, 'eggs = False') + + self.assertIn('spam', ns1) + self.assertNotIn('eggs', ns1) + self.assertIn('eggs', ns2) + self.assertNotIn('spam', ns2) + def test_return_execution_namespace(self): script = dedent(""" spam = 42 diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 6366c7ec5e7611..a5146cda48af15 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -89,6 +89,33 @@ _ensure_not_running(PyInterpreterState *interp) return 0; } +static PyObject * +_copy_module(PyObject *m, PyObject *updates) +{ + PyObject *orig = PyModule_GetDict(m); // borrowed + if (orig == NULL) { + return NULL; + } + + PyObject *copy = PyModule_New("__main__"); + if (copy == NULL) { + return NULL; + } + PyObject *ns = PyModule_GetDict(copy); // borrowed + if (ns == NULL) + goto error; + + if (PyDict_Merge(ns, orig, 1) < 0) + goto error; + if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) + goto error; + return copy; + +error: + Py_DECREF(copy); + return NULL; +} + static int _run_string(PyInterpreterState *interp, const char *codestr, PyObject *ns) { @@ -123,31 +150,46 @@ _run_string_in_main(PyInterpreterState *interp, const char *codestr, if (_ensure_not_running(interp) < 0) return NULL; - // Get the namespace in which to execute. - // XXX Force a fresh __main__ module? - PyObject *m = PyMapping_GetItemString(interp->modules, "__main__"); - if (m == NULL) + // Get the namespace in which to execute. This involves creating + // a new module, updating it from the __main__ module and the given + // updates (if any), replacing the __main__ with the new module in + // sys.modules, and then using the new module's __dict__. At the + // end we restore the original __main__ module. + PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); + if (main_mod == NULL) return NULL; - PyObject *orig = PyModule_GetDict(m); // borrowed - Py_DECREF(m); - if (orig == NULL) - return NULL; - PyObject *ns = PyDict_Copy(orig); - if (ns == NULL) + PyObject *m = _copy_module(main_mod, updates); + if (m == NULL) { + Py_DECREF(main_mod); return NULL; - if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) { - Py_DECREF(ns); + } + if (PyMapping_SetItemString(interp->modules, "__main__", m) < 0) { + Py_DECREF(main_mod); + Py_DECREF(m); return NULL; } + PyObject *ns = PyModule_GetDict(m); // borrowed + Py_INCREF(ns); + Py_DECREF(m); + // Run the string. + PyObject *result = ns; if (_run_string(interp, codestr, ns) < 0) { + result = NULL; Py_DECREF(ns); - return NULL; } - return ns; + // Restore __main__. + if (PyMapping_SetItemString(interp->modules, "__main__", main_mod) < 0) { + // XXX Chain exceptions... + //PyErr_Restore(exc, value, tb); + } + Py_DECREF(main_mod); + + return result; } + /* module level code ********************************************************/ static PyObject * From ec05cf5f6f9cf32d1cc5992a579e6db105831ad0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 11:58:31 -0700 Subject: [PATCH 12/28] Close stdin and stdout after the proc finishes. --- Lib/test/test__interpreters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 3012f4e59ea83d..93c1ee58a7e19f 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -54,8 +54,8 @@ def test_still_running_at_exit(self): time.sleep(1_000_000) """) filename = script_helper.make_script(self.dirname, 'interp', script) - proc = script_helper.spawn_python(filename) - retcode = proc.wait() + with script_helper.spawn_python(filename) as proc: + retcode = proc.wait() self.assertEqual(retcode, 0) From fe9046614fbf4f73b5e08783d2683061aee162c7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 12:12:07 -0700 Subject: [PATCH 13/28] Clean up a test. --- Lib/test/test__interpreters.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 93c1ee58a7e19f..459ff8650a8470 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -3,7 +3,7 @@ import os.path import shutil import tempfile -from textwrap import dedent +from textwrap import dedent, indent import threading import unittest @@ -14,10 +14,13 @@ SCRIPT_THREADED_INTERP = """\ +from textwrap import dedent import threading import _interpreters def f(): - _interpreters.run_string(id, {!r}) + _interpreters.run_string(id, dedent(''' + {} + ''')) t = threading.Thread(target=f) t.start() @@ -48,11 +51,12 @@ def tearDown(self): shutil.rmtree(self.dirname) def test_still_running_at_exit(self): - script = SCRIPT_THREADED_INTERP.format("""if True: + subscript = dedent(""" import time # Give plenty of time for the main interpreter to finish. time.sleep(1_000_000) """) + script = SCRIPT_THREADED_INTERP.format(indent(subscript, ' ')) filename = script_helper.make_script(self.dirname, 'interp', script) with script_helper.spawn_python(filename) as proc: retcode = proc.wait() From 90820179ef8aae5ebd3bd03691cc29b11ab25cc0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 13:10:53 -0700 Subject: [PATCH 14/28] Chain exceptions during cleanup. --- Modules/_interpretersmodule.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index a5146cda48af15..ff08ae28c449dd 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -180,9 +180,12 @@ _run_string_in_main(PyInterpreterState *interp, const char *codestr, } // Restore __main__. + PyObject *exc = NULL, *value = NULL, *tb = NULL; + PyErr_Fetch(&exc, &value, &tb); if (PyMapping_SetItemString(interp->modules, "__main__", main_mod) < 0) { - // XXX Chain exceptions... - //PyErr_Restore(exc, value, tb); + _PyErr_ChainExceptions(exc, value, tb); + } else { + PyErr_Restore(exc, value, tb); } Py_DECREF(main_mod); From 21865d4c0ff85ad92da8e08229e7514c12c07fb8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 13:31:50 -0700 Subject: [PATCH 15/28] Finish the module docs. --- Doc/library/_interpreters.rst | 36 +++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index e2d0191ac71703..6e79277895936e 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -6,21 +6,30 @@ .. versionadded:: 3,7 - :ref:`_sub-interpreter-support` - -threading - -------------- This module provides low-level primitives for working with multiple -Python interpreters in the same process. - +Python interpreters in the same runtime in the current process. .. XXX The :mod:`interpreters` module provides an easier to use and higher-level API built on top of this module. +More information about (sub)interpreters is found at +:ref:`_sub-interpreter-support`, including what data is shared between +interpreters and what is unique. Note particularly that interpreters +aren't inherently threaded, even though they track and manage Python +threads. To run code in an interpreter in a different OS thread, call +:func:`run_string` in a function that you run in a new Python thread. +For example:: + + id = _interpreters.create() + def f(): + _interpreters.run_string(id, 'print("in a thread")') + + t = threading.Thread(target=f) + t.start() + This module is optional. It is provided by Python implementations which support multiple interpreters. - .. XXX For systems lacking the :mod:`_interpreters` module, the :mod:`_dummy_interpreters` module is available. It duplicates this module's interface and can be used as a drop-in replacement. @@ -42,9 +51,10 @@ It defines the following functions: .. function:: run_string(id, command) A wrapper around :c:func:`PyRun_SimpleString` which runs the provided - Python program using the identified interpreter. Providing an - invalid or unknown ID results in a RuntimeError, likewise if the main - interpreter or any other running interpreter is used. + Python program in the main thread of the identified interpreter. + Providing an invalid or unknown ID results in a RuntimeError, + likewise if the main interpreter or any other running interpreter + is used. Any value returned from the code is thrown away, similar to what threads do. If the code results in an exception then that exception @@ -62,9 +72,3 @@ It defines the following functions: merged into the execution namespace before execution. Note that this allows objects to leak between interpreters, which may not be desirable. - - -**Caveats:** - -* ... - From 0342a4f92b82e962983bf6d6d0c240631dc630f7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 23 May 2017 16:41:55 -0700 Subject: [PATCH 16/28] Fix docs. --- Doc/library/_interpreters.rst | 7 +------ Doc/library/concurrency.rst | 1 + 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 6e79277895936e..445926ac4b87ba 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -10,11 +10,9 @@ This module provides low-level primitives for working with multiple Python interpreters in the same runtime in the current process. -.. XXX The :mod:`interpreters` module provides an easier to use and - higher-level API built on top of this module. More information about (sub)interpreters is found at -:ref:`_sub-interpreter-support`, including what data is shared between +:ref:`sub-interpreter-support`, including what data is shared between interpreters and what is unique. Note particularly that interpreters aren't inherently threaded, even though they track and manage Python threads. To run code in an interpreter in a different OS thread, call @@ -30,9 +28,6 @@ For example:: This module is optional. It is provided by Python implementations which support multiple interpreters. -.. XXX For systems lacking the :mod:`_interpreters` module, the - :mod:`_dummy_interpreters` module is available. It duplicates this - module's interface and can be used as a drop-in replacement. It defines the following functions: diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index 826bf86d081793..fafbf92c6b0181 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -29,3 +29,4 @@ The following are support modules for some of the above services: _thread.rst _dummy_thread.rst dummy_threading.rst + _interpreters.rst From e9d9b04e0ca0b6d34f9bf92c069ea967d1e061b3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Dec 2017 18:28:54 -0700 Subject: [PATCH 17/28] Fix includes. --- Modules/_interpretersmodule.c | 1 + setup.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index ff08ae28c449dd..64a9e7ff6f6ad7 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -4,6 +4,7 @@ #include "Python.h" #include "frameobject.h" +#include "internal/pystate.h" static PyInterpreterState * diff --git a/setup.py b/setup.py index 531e237b66aa1f..eaf3ba218266c0 100644 --- a/setup.py +++ b/setup.py @@ -741,7 +741,8 @@ def detect_modules(self): ) # Python interface to subinterpreter C-API. - exts.append(Extension('_interpreters', ['_interpretersmodule.c'])) + exts.append(Extension('_interpreters', ['_interpretersmodule.c'], + define_macros=[('Py_BUILD_CORE', '')])) # # Here ends the simple stuff. From here on, modules need certain From 722ae943150387f19aaa139ee403c4f16d1f69cb Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 28 Nov 2017 12:44:47 -0600 Subject: [PATCH 18/28] Add _interpreters.is_shareable(). --- Modules/_interpretersmodule.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 64a9e7ff6f6ad7..825f987fdf4f2b 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -397,6 +397,26 @@ merged into the execution namespace before the code is executed.\n\ See PyRun_SimpleStrings."); +static PyObject * +object_is_shareable(PyObject *self, PyObject *args) +{ + PyObject *obj; + if (!PyArg_UnpackTuple(args, "is_shareable", 1, 1, &obj)) + return NULL; + + if (PyBytes_CheckExact(obj)) + Py_RETURN_TRUE; + + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(is_shareable_doc, +"is_shareable(obj) -> bool\n\ +\n\ +Return True if the object's data may be shared between interpreters and\n\ +False otherwise."); + + static PyMethodDef module_functions[] = { {"create", (PyCFunction)interp_create, METH_VARARGS, create_doc}, @@ -411,6 +431,9 @@ static PyMethodDef module_functions[] = { {"run_string_unrestricted", (PyCFunction)interp_run_string_unrestricted, METH_VARARGS, run_string_unrestricted_doc}, + {"is_shareable", (PyCFunction)object_is_shareable, + METH_VARARGS, is_shareable_doc}, + {NULL, NULL} /* sentinel */ }; From 061ae139883197878bbef6e60eea226afebb2c2e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Dec 2017 15:59:00 -0700 Subject: [PATCH 19/28] Add _PyObject_CheckShareable(). --- Modules/_interpretersmodule.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 825f987fdf4f2b..b47ff6212af17f 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -7,6 +7,20 @@ #include "internal/pystate.h" +/* sharing-specific functions */ + +static int +_PyObject_CheckShareable(PyObject *obj) +{ + if (PyBytes_CheckExact(obj)) + return 0; + PyErr_SetString(PyExc_ValueError, + "obj is not a cross-interpreter shareable type"); + return 1; +} + +/* interpreter-specific functions */ + static PyInterpreterState * _get_current(void) { @@ -403,10 +417,9 @@ object_is_shareable(PyObject *self, PyObject *args) PyObject *obj; if (!PyArg_UnpackTuple(args, "is_shareable", 1, 1, &obj)) return NULL; - - if (PyBytes_CheckExact(obj)) + if (_PyObject_CheckShareable(obj) == 0) Py_RETURN_TRUE; - + PyErr_Clear(); Py_RETURN_FALSE; } From 9a365ecd9e58393ab3f7198290357f471168e509 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 4 Dec 2017 16:50:55 -0700 Subject: [PATCH 20/28] Add _PyCrossInterpreterData. --- Include/internal/pystate.h | 14 ++++++ Modules/_interpretersmodule.c | 85 +++++++++++++++++++++++++++++++---- 2 files changed, 91 insertions(+), 8 deletions(-) diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index b93342120477f3..7816665bb00df5 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -80,6 +80,20 @@ PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate( PyAPI_FUNC(void) _PyPathConfig_Clear(_PyPathConfig *config); +/* Cross-interpreter data sharing */ + +struct _cid; + +typedef struct _cid { + void *data; + PyObject *(*new_object)(struct _cid *); + void (*free)(void *); + + PyInterpreterState *interp; + PyObject *object; +} _PyCrossInterpreterData; + + /* Full Python runtime state */ typedef struct pyruntimestate { diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index b47ff6212af17f..eb7f444cdf6ad1 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -7,6 +7,17 @@ #include "internal/pystate.h" +static PyInterpreterState * +_get_current(void) +{ + PyThreadState *tstate; + + tstate = PyThreadState_Get(); + if (tstate == NULL) + return NULL; + return tstate->interp; +} + /* sharing-specific functions */ static int @@ -19,19 +30,77 @@ _PyObject_CheckShareable(PyObject *obj) return 1; } -/* interpreter-specific functions */ +static PyObject * +_new_bytes_object(_PyCrossInterpreterData *data) +{ + return PyBytes_FromString((char *)(data->data)); +} -static PyInterpreterState * -_get_current(void) +static int +_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data) { - PyThreadState *tstate; + data->data = (void *)(PyBytes_AS_STRING(obj)); + data->new_object = _new_bytes_object; + data->free = NULL; + return 0; +} - tstate = PyThreadState_Get(); - if (tstate == NULL) - return NULL; - return tstate->interp; +static int +_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) +{ + Py_INCREF(obj); + + if (_PyObject_CheckShareable(obj) != 0) { + Py_DECREF(obj); + return 1; + } + + data->interp = _get_current(); + data->object = obj; + + if (PyBytes_CheckExact(obj)) { + return _bytes_shared(obj, data); + } + + return 0; +}; + +static void +_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) +{ + PyThreadState *save_tstate = NULL; + if (data->interp != NULL) { + // Switch to the original interpreter. + PyThreadState *tstate = PyInterpreterState_ThreadHead(data->interp); + save_tstate = PyThreadState_Swap(tstate); + } + + if (data->free != NULL) { + data->free(data->data); + } + Py_XDECREF(data->object); + + // Switch back. + if (save_tstate != NULL) + PyThreadState_Swap(save_tstate); +} + +static PyObject * +_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) +{ + return data->new_object(data); } +static PyObject * +_PyCrossInterpreterData_Use(_PyCrossInterpreterData *data) +{ + PyObject *obj = _PyCrossInterpreterData_NewObject(data); + _PyCrossInterpreterData_Release(data); + return obj; +} + +/* interpreter-specific functions */ + static PyInterpreterState * _look_up_int64(PY_INT64_T requested_id) { From 8f299d42e3b25b8b150845b6e99680414f8de87b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 7 Dec 2017 16:28:11 -0700 Subject: [PATCH 21/28] Use the shared data in run() safely. --- Modules/_interpretersmodule.c | 289 ++++++++++++++++++++++++++-------- 1 file changed, 221 insertions(+), 68 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index eb7f444cdf6ad1..11ea2479a6cf42 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -59,7 +59,10 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) data->object = obj; if (PyBytes_CheckExact(obj)) { - return _bytes_shared(obj, data); + if (_bytes_shared(obj, data) != 0) { + Py_DECREF(obj); + return 1; + } } return 0; @@ -91,14 +94,6 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) return data->new_object(data); } -static PyObject * -_PyCrossInterpreterData_Use(_PyCrossInterpreterData *data) -{ - PyObject *obj = _PyCrossInterpreterData_NewObject(data); - _PyCrossInterpreterData_Release(data); - return obj; -} - /* interpreter-specific functions */ static PyInterpreterState * @@ -174,14 +169,14 @@ _ensure_not_running(PyInterpreterState *interp) } static PyObject * -_copy_module(PyObject *m, PyObject *updates) +_copy_module(PyObject *m, const char *name) { PyObject *orig = PyModule_GetDict(m); // borrowed if (orig == NULL) { return NULL; } - PyObject *copy = PyModule_New("__main__"); + PyObject *copy = PyModule_New(name); if (copy == NULL) { return NULL; } @@ -191,8 +186,6 @@ _copy_module(PyObject *m, PyObject *updates) if (PyDict_Merge(ns, orig, 1) < 0) goto error; - if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) - goto error; return copy; error: @@ -200,78 +193,235 @@ _copy_module(PyObject *m, PyObject *updates) return NULL; } -static int -_run_string(PyInterpreterState *interp, const char *codestr, PyObject *ns) -{ - // Switch to interpreter. - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); - PyThreadState *save_tstate = PyThreadState_Swap(tstate); - - // Run the string (see PyRun_SimpleStringFlags). - PyObject *exc = NULL, *value = NULL, *tb = NULL; - PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); - if (result == NULL) { - // Get the exception from the subinterpreter. - PyErr_Fetch(&exc, &value, &tb); - } else { - Py_DECREF(result); // We throw away the result. - } - - // Switch back. - if (save_tstate != NULL) - PyThreadState_Swap(save_tstate); - - // Propagate any exception out to the caller. - PyErr_Restore(exc, value, tb); - - return result == NULL ? -1 : 0; -} - static PyObject * -_run_string_in_main(PyInterpreterState *interp, const char *codestr, - PyObject *updates) +_copy_module_ns(PyInterpreterState *interp, + const char *name, const char *tempname) { - if (_ensure_not_running(interp) < 0) - return NULL; - // Get the namespace in which to execute. This involves creating // a new module, updating it from the __main__ module and the given // updates (if any), replacing the __main__ with the new module in // sys.modules, and then using the new module's __dict__. At the // end we restore the original __main__ module. - PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); + PyObject *main_mod = PyMapping_GetItemString(interp->modules, name); if (main_mod == NULL) return NULL; - PyObject *m = _copy_module(main_mod, updates); + PyObject *m = _copy_module(main_mod, name); if (m == NULL) { Py_DECREF(main_mod); return NULL; } - if (PyMapping_SetItemString(interp->modules, "__main__", m) < 0) { - Py_DECREF(main_mod); + if (tempname != NULL) { + if (PyMapping_SetItemString(interp->modules, tempname, main_mod) < 0) { + Py_DECREF(main_mod); + Py_DECREF(m); + return NULL; + } + } + Py_DECREF(main_mod); + if (PyMapping_SetItemString(interp->modules, name, m) < 0) { Py_DECREF(m); return NULL; } PyObject *ns = PyModule_GetDict(m); // borrowed Py_INCREF(ns); Py_DECREF(m); + return ns; +} - // Run the string. - PyObject *result = ns; - if (_run_string(interp, codestr, ns) < 0) { - result = NULL; - Py_DECREF(ns); +static int +_restore_module(PyInterpreterState *interp, + const char *name, const char *tempname) +{ + PyObject *main_mod = PyMapping_GetItemString(interp->modules, tempname); + if (main_mod == NULL) + return -1; + if (PyMapping_SetItemString(interp->modules, name, main_mod) < 0) { + Py_DECREF(main_mod); + return -1; } + Py_DECREF(main_mod); + return 0; +} - // Restore __main__. - PyObject *exc = NULL, *value = NULL, *tb = NULL; - PyErr_Fetch(&exc, &value, &tb); - if (PyMapping_SetItemString(interp->modules, "__main__", main_mod) < 0) { - _PyErr_ChainExceptions(exc, value, tb); +struct _shareditem { + Py_UNICODE *name; + Py_ssize_t namelen; + _PyCrossInterpreterData data; +}; + +void +_sharedns_clear(struct _shareditem *shared) +{ + for (struct _shareditem *item=shared; item->name != NULL; item += 1) { + _PyCrossInterpreterData_Release(&item->data); + } +} + +static struct _shareditem * +_get_shared_ns(PyObject *shareable) +{ + if (shareable == NULL || shareable == Py_None) + return NULL; + Py_ssize_t len = PyDict_Size(shareable); + if (len == 0) + return NULL; + + struct _shareditem *shared = PyMem_NEW(struct _shareditem, len+1); + Py_ssize_t pos = 0; + for (Py_ssize_t i=0; i < len; i++) { + PyObject *key, *value; + if (PyDict_Next(shareable, &pos, &key, &value) == 0) { + break; + } + struct _shareditem *item = shared + i; + + if (_PyObject_GetCrossInterpreterData(value, &item->data) != 0) + break; + item->name = PyUnicode_AsUnicodeAndSize(key, &item->namelen); + if (item->name == NULL) { + _PyCrossInterpreterData_Release(&item->data); + break; + } + (item + 1)->name = NULL; // Mark the next one as the last. + } + if (PyErr_Occurred()) { + _sharedns_clear(shared); + PyMem_Free(shared); + return NULL; + } + return shared; +} + +static int +_shareditem_apply(struct _shareditem *item, PyObject *ns) +{ + PyObject *name = PyUnicode_FromUnicode(item->name, item->namelen); + if (name == NULL) { + return 1; + } + PyObject *value = _PyCrossInterpreterData_NewObject(&item->data); + if (value == NULL) { + Py_DECREF(name); + return 1; + } + int res = PyDict_SetItem(ns, name, value); + Py_DECREF(name); + Py_DECREF(value); + return res; +} + +// XXX This cannot use PyObject fields. + +struct _shared_exception { + PyObject *exc; + PyObject *value; + PyObject *tb; +}; + +static struct _shared_exception * +_get_shared_exception(void) +{ + struct _shared_exception *exc = PyMem_NEW(struct _shared_exception, 1); + // XXX Fatal if NULL? + PyErr_Fetch(&exc->exc, &exc->value, &exc->tb); + return exc; +} + +static void +_apply_shared_exception(struct _shared_exception *exc) +{ + if (PyErr_Occurred()) { + _PyErr_ChainExceptions(exc->exc, exc->value, exc->tb); } else { - PyErr_Restore(exc, value, tb); + PyErr_Restore(exc->exc, exc->value, exc->tb); + } + +} + +// XXX Return int instead. + +static PyObject * +_run_script(PyInterpreterState *interp, const char *codestr, + struct _shareditem *shared, struct _shared_exception **exc) +{ + // XXX Do not copy. + + // Get a copy of the __main__ module. + // + // This involves creating a new module, updating it from the + // __main__ module and the given updates (if any), replacing the + // __main__ with the new module in sys.modules, and then using the + // new module's __dict__. At the end we restore the original + // __main__ module. + PyObject *ns = _copy_module_ns(interp, "__main__", "_orig___main___"); + if (ns == NULL) { + return NULL; + } + + // Apply the cross-interpreter data. + if (shared != NULL) { + for (struct _shareditem *item=shared; item->name != NULL; item += 1) { + if (_shareditem_apply(shared, ns) != 0) { + Py_DECREF(ns); + ns = NULL; + goto done; + } + } + } + + // Run the string (see PyRun_SimpleStringFlags). + PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); + if (result == NULL) { + // Get the exception from the subinterpreter. + *exc = _get_shared_exception(); + // XXX Clear the exception? + } else { + Py_DECREF(result); // We throw away the result. + } + +done: + // Restore __main__. + if (_restore_module(interp, "__main__", "_orig___main___") != 0) { + // XXX How to propagate this exception... + //_PyErr_ChainExceptions(exc, value, tb); + } + return ns; +} + +static PyObject * +_run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, + PyObject *shareable) +{ + // XXX lock? + if (_ensure_not_running(interp) < 0) + return NULL; + + struct _shareditem *shared = _get_shared_ns(shareable); + if (shared == NULL && PyErr_Occurred()) + return NULL; + + // Switch to interpreter. + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + PyThreadState *save_tstate = PyThreadState_Swap(tstate); + + // Run the script. + struct _shared_exception *exc = NULL; + PyObject *result = _run_script(interp, codestr, shared, &exc); + // XXX What to do if result is NULL? + + // Switch back. + if (save_tstate != NULL) + PyThreadState_Swap(save_tstate); + + // Propagate any exception out to the caller. + if (exc != NULL) { + _apply_shared_exception(exc); + } + + if (shared != NULL) { + _sharedns_clear(shared); + PyMem_Free(shared); } - Py_DECREF(main_mod); return result; } @@ -387,7 +537,8 @@ static PyObject * interp_run_string(PyObject *self, PyObject *args) { PyObject *id, *code; - if (!PyArg_UnpackTuple(args, "run_string", 2, 2, &id, &code)) + PyObject *shared = NULL; + if (!PyArg_UnpackTuple(args, "run_string", 2, 3, &id, &code, &shared)) return NULL; if (!PyLong_Check(id)) { PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int"); @@ -416,7 +567,7 @@ interp_run_string(PyObject *self, PyObject *args) } // Run the code in the interpreter. - PyObject *ns = _run_string_in_main(interp, codestr, NULL); + PyObject *ns = _run_script_in_interpreter(interp, codestr, shared); if (ns == NULL) return NULL; else @@ -431,12 +582,14 @@ Execute the provided string in the identified interpreter.\n\ See PyRun_SimpleStrings."); +/* XXX Drop run_string_unrestricted(). */ + static PyObject * interp_run_string_unrestricted(PyObject *self, PyObject *args) { - PyObject *id, *code, *ns = NULL; + PyObject *id, *code, *shared = NULL; if (!PyArg_UnpackTuple(args, "run_string_unrestricted", 2, 3, - &id, &code, &ns)) + &id, &code, &shared)) return NULL; if (!PyLong_Check(id)) { PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int"); @@ -447,8 +600,8 @@ interp_run_string_unrestricted(PyObject *self, PyObject *args) "second arg (code) must be a string"); return NULL; } - if (ns == Py_None) - ns = NULL; + if (shared == Py_None) + shared = NULL; // Look up the interpreter. PyInterpreterState *interp = _look_up(id); @@ -467,7 +620,7 @@ interp_run_string_unrestricted(PyObject *self, PyObject *args) } // Run the code in the interpreter. - return _run_string_in_main(interp, codestr, ns); + return _run_script_in_interpreter(interp, codestr, shared); } PyDoc_STRVAR(run_string_unrestricted_doc, From 92029a12c8dac316a338347d1c54c8a068501b5e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 7 Dec 2017 16:56:35 -0700 Subject: [PATCH 22/28] Do not use a copy of the __main__ ns. --- Modules/_interpretersmodule.c | 102 +++------------------------------- 1 file changed, 7 insertions(+), 95 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 11ea2479a6cf42..26b7f69ed71940 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -168,81 +168,6 @@ _ensure_not_running(PyInterpreterState *interp) return 0; } -static PyObject * -_copy_module(PyObject *m, const char *name) -{ - PyObject *orig = PyModule_GetDict(m); // borrowed - if (orig == NULL) { - return NULL; - } - - PyObject *copy = PyModule_New(name); - if (copy == NULL) { - return NULL; - } - PyObject *ns = PyModule_GetDict(copy); // borrowed - if (ns == NULL) - goto error; - - if (PyDict_Merge(ns, orig, 1) < 0) - goto error; - return copy; - -error: - Py_DECREF(copy); - return NULL; -} - -static PyObject * -_copy_module_ns(PyInterpreterState *interp, - const char *name, const char *tempname) -{ - // Get the namespace in which to execute. This involves creating - // a new module, updating it from the __main__ module and the given - // updates (if any), replacing the __main__ with the new module in - // sys.modules, and then using the new module's __dict__. At the - // end we restore the original __main__ module. - PyObject *main_mod = PyMapping_GetItemString(interp->modules, name); - if (main_mod == NULL) - return NULL; - PyObject *m = _copy_module(main_mod, name); - if (m == NULL) { - Py_DECREF(main_mod); - return NULL; - } - if (tempname != NULL) { - if (PyMapping_SetItemString(interp->modules, tempname, main_mod) < 0) { - Py_DECREF(main_mod); - Py_DECREF(m); - return NULL; - } - } - Py_DECREF(main_mod); - if (PyMapping_SetItemString(interp->modules, name, m) < 0) { - Py_DECREF(m); - return NULL; - } - PyObject *ns = PyModule_GetDict(m); // borrowed - Py_INCREF(ns); - Py_DECREF(m); - return ns; -} - -static int -_restore_module(PyInterpreterState *interp, - const char *name, const char *tempname) -{ - PyObject *main_mod = PyMapping_GetItemString(interp->modules, tempname); - if (main_mod == NULL) - return -1; - if (PyMapping_SetItemString(interp->modules, name, main_mod) < 0) { - Py_DECREF(main_mod); - return -1; - } - Py_DECREF(main_mod); - return 0; -} - struct _shareditem { Py_UNICODE *name; Py_ssize_t namelen; @@ -344,27 +269,20 @@ static PyObject * _run_script(PyInterpreterState *interp, const char *codestr, struct _shareditem *shared, struct _shared_exception **exc) { - // XXX Do not copy. - - // Get a copy of the __main__ module. - // - // This involves creating a new module, updating it from the - // __main__ module and the given updates (if any), replacing the - // __main__ with the new module in sys.modules, and then using the - // new module's __dict__. At the end we restore the original - // __main__ module. - PyObject *ns = _copy_module_ns(interp, "__main__", "_orig___main___"); - if (ns == NULL) { + PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); + if (main_mod == NULL) + return NULL; + PyObject *ns = PyModule_GetDict(main_mod); // borrowed + Py_DECREF(main_mod); + if (ns == NULL) return NULL; - } // Apply the cross-interpreter data. if (shared != NULL) { for (struct _shareditem *item=shared; item->name != NULL; item += 1) { if (_shareditem_apply(shared, ns) != 0) { Py_DECREF(ns); - ns = NULL; - goto done; + return NULL; } } } @@ -379,12 +297,6 @@ _run_script(PyInterpreterState *interp, const char *codestr, Py_DECREF(result); // We throw away the result. } -done: - // Restore __main__. - if (_restore_module(interp, "__main__", "_orig___main___") != 0) { - // XXX How to propagate this exception... - //_PyErr_ChainExceptions(exc, value, tb); - } return ns; } From 52c9c2fa237cdbf00e4f8fc8a0eb3e2c80242e02 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 7 Dec 2017 17:04:35 -0700 Subject: [PATCH 23/28] Never return the execution namespace. --- Modules/_interpretersmodule.c | 85 +++++++---------------------------- 1 file changed, 15 insertions(+), 70 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 26b7f69ed71940..324b25f6a187dc 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -263,32 +263,31 @@ _apply_shared_exception(struct _shared_exception *exc) } -// XXX Return int instead. - -static PyObject * +static int _run_script(PyInterpreterState *interp, const char *codestr, struct _shareditem *shared, struct _shared_exception **exc) { PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); if (main_mod == NULL) - return NULL; + return -1; PyObject *ns = PyModule_GetDict(main_mod); // borrowed Py_DECREF(main_mod); if (ns == NULL) - return NULL; + return -1; // Apply the cross-interpreter data. if (shared != NULL) { for (struct _shareditem *item=shared; item->name != NULL; item += 1) { if (_shareditem_apply(shared, ns) != 0) { Py_DECREF(ns); - return NULL; + return -1; } } } // Run the string (see PyRun_SimpleStringFlags). PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); + Py_DECREF(ns); if (result == NULL) { // Get the exception from the subinterpreter. *exc = _get_shared_exception(); @@ -297,20 +296,20 @@ _run_script(PyInterpreterState *interp, const char *codestr, Py_DECREF(result); // We throw away the result. } - return ns; + return 0; } -static PyObject * +static int _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, PyObject *shareable) { // XXX lock? if (_ensure_not_running(interp) < 0) - return NULL; + return -1; struct _shareditem *shared = _get_shared_ns(shareable); if (shared == NULL && PyErr_Occurred()) - return NULL; + return -1; // Switch to interpreter. PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); @@ -318,8 +317,9 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, // Run the script. struct _shared_exception *exc = NULL; - PyObject *result = _run_script(interp, codestr, shared, &exc); - // XXX What to do if result is NULL? + if (_run_script(interp, codestr, shared, &exc) != 0) { + // XXX What to do if the the result isn't 0? + } // Switch back. if (save_tstate != NULL) @@ -335,7 +335,7 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, PyMem_Free(shared); } - return result; + return 0; } @@ -479,11 +479,9 @@ interp_run_string(PyObject *self, PyObject *args) } // Run the code in the interpreter. - PyObject *ns = _run_script_in_interpreter(interp, codestr, shared); - if (ns == NULL) + if (_run_script_in_interpreter(interp, codestr, shared) != 0) return NULL; - else - Py_RETURN_NONE; + Py_RETURN_NONE; } PyDoc_STRVAR(run_string_doc, @@ -494,57 +492,6 @@ Execute the provided string in the identified interpreter.\n\ See PyRun_SimpleStrings."); -/* XXX Drop run_string_unrestricted(). */ - -static PyObject * -interp_run_string_unrestricted(PyObject *self, PyObject *args) -{ - PyObject *id, *code, *shared = NULL; - if (!PyArg_UnpackTuple(args, "run_string_unrestricted", 2, 3, - &id, &code, &shared)) - return NULL; - if (!PyLong_Check(id)) { - PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int"); - return NULL; - } - if (!PyUnicode_Check(code)) { - PyErr_SetString(PyExc_TypeError, - "second arg (code) must be a string"); - return NULL; - } - if (shared == Py_None) - shared = NULL; - - // Look up the interpreter. - PyInterpreterState *interp = _look_up(id); - if (interp == NULL) - return NULL; - - // Extract code. - Py_ssize_t size; - const char *codestr = PyUnicode_AsUTF8AndSize(code, &size); - if (codestr == NULL) - return NULL; - if (strlen(codestr) != (size_t)size) { - PyErr_SetString(PyExc_ValueError, - "source code string cannot contain null bytes"); - return NULL; - } - - // Run the code in the interpreter. - return _run_script_in_interpreter(interp, codestr, shared); -} - -PyDoc_STRVAR(run_string_unrestricted_doc, -"run_string_unrestricted(ID, sourcetext, ns=None) -> main module ns\n\ -\n\ -Execute the provided string in the identified interpreter. Return the\n\ -dict in which the code executed. If the ns arg is provided then it is\n\ -merged into the execution namespace before the code is executed.\n\ -\n\ -See PyRun_SimpleStrings."); - - static PyObject * object_is_shareable(PyObject *self, PyObject *args) { @@ -575,8 +522,6 @@ static PyMethodDef module_functions[] = { {"run_string", (PyCFunction)interp_run_string, METH_VARARGS, run_string_doc}, - {"run_string_unrestricted", (PyCFunction)interp_run_string_unrestricted, - METH_VARARGS, run_string_unrestricted_doc}, {"is_shareable", (PyCFunction)object_is_shareable, METH_VARARGS, is_shareable_doc}, From a797883ca2511aeeeb0393ac80b357f23e83efff Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 7 Dec 2017 17:09:03 -0700 Subject: [PATCH 24/28] Group sharing-related code. --- Modules/_interpretersmodule.c | 150 +++++++++++++++++----------------- 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 324b25f6a187dc..7909cef65c0b7e 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -18,7 +18,7 @@ _get_current(void) return tstate->interp; } -/* sharing-specific functions */ +/* sharing-specific functions and structs */ static int _PyObject_CheckShareable(PyObject *obj) @@ -94,80 +94,6 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) return data->new_object(data); } -/* interpreter-specific functions */ - -static PyInterpreterState * -_look_up_int64(PY_INT64_T requested_id) -{ - if (requested_id < 0) - goto error; - - PyInterpreterState *interp = PyInterpreterState_Head(); - while (interp != NULL) { - PY_INT64_T id = PyInterpreterState_GetID(interp); - if (id < 0) - return NULL; - if (requested_id == id) - return interp; - interp = PyInterpreterState_Next(interp); - } - -error: - PyErr_Format(PyExc_RuntimeError, - "unrecognized interpreter ID %lld", requested_id); - return NULL; -} - -static PyInterpreterState * -_look_up(PyObject *requested_id) -{ - long long id = PyLong_AsLongLong(requested_id); - if (id == -1 && PyErr_Occurred() != NULL) - return NULL; - assert(id <= INT64_MAX); - return _look_up_int64(id); -} - -static PyObject * -_get_id(PyInterpreterState *interp) -{ - PY_INT64_T id = PyInterpreterState_GetID(interp); - if (id < 0) - return NULL; - return PyLong_FromLongLong(id); -} - -static int -_is_running(PyInterpreterState *interp) -{ - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); - if (PyThreadState_Next(tstate) != NULL) { - PyErr_SetString(PyExc_RuntimeError, - "interpreter has more than one thread"); - return -1; - } - PyFrameObject *frame = _PyThreadState_GetFrame(tstate); - if (frame == NULL) { - if (PyErr_Occurred() != NULL) - return -1; - return 0; - } - return (int)(frame->f_executing); -} - -static int -_ensure_not_running(PyInterpreterState *interp) -{ - int is_running = _is_running(interp); - if (is_running < 0) - return -1; - if (is_running) { - PyErr_Format(PyExc_RuntimeError, "interpreter already running"); - return -1; - } - return 0; -} - struct _shareditem { Py_UNICODE *name; Py_ssize_t namelen; @@ -263,6 +189,80 @@ _apply_shared_exception(struct _shared_exception *exc) } +/* interpreter-specific functions */ + +static PyInterpreterState * +_look_up_int64(PY_INT64_T requested_id) +{ + if (requested_id < 0) + goto error; + + PyInterpreterState *interp = PyInterpreterState_Head(); + while (interp != NULL) { + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) + return NULL; + if (requested_id == id) + return interp; + interp = PyInterpreterState_Next(interp); + } + +error: + PyErr_Format(PyExc_RuntimeError, + "unrecognized interpreter ID %lld", requested_id); + return NULL; +} + +static PyInterpreterState * +_look_up(PyObject *requested_id) +{ + long long id = PyLong_AsLongLong(requested_id); + if (id == -1 && PyErr_Occurred() != NULL) + return NULL; + assert(id <= INT64_MAX); + return _look_up_int64(id); +} + +static PyObject * +_get_id(PyInterpreterState *interp) +{ + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) + return NULL; + return PyLong_FromLongLong(id); +} + +static int +_is_running(PyInterpreterState *interp) +{ + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + if (PyThreadState_Next(tstate) != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "interpreter has more than one thread"); + return -1; + } + PyFrameObject *frame = _PyThreadState_GetFrame(tstate); + if (frame == NULL) { + if (PyErr_Occurred() != NULL) + return -1; + return 0; + } + return (int)(frame->f_executing); +} + +static int +_ensure_not_running(PyInterpreterState *interp) +{ + int is_running = _is_running(interp); + if (is_running < 0) + return -1; + if (is_running) { + PyErr_Format(PyExc_RuntimeError, "interpreter already running"); + return -1; + } + return 0; +} + static int _run_script(PyInterpreterState *interp, const char *codestr, struct _shareditem *shared, struct _shared_exception **exc) From 777838a31bac3e758be4b422023233c34a183e42 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 7 Dec 2017 17:10:45 -0700 Subject: [PATCH 25/28] Fix a refcount. --- Modules/_interpretersmodule.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 7909cef65c0b7e..299e71e620ddc3 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -271,6 +271,7 @@ _run_script(PyInterpreterState *interp, const char *codestr, if (main_mod == NULL) return -1; PyObject *ns = PyModule_GetDict(main_mod); // borrowed + Py_INCREF(ns); Py_DECREF(main_mod); if (ns == NULL) return -1; From ab8f1755f33940b098810c2ff8aa62889d2b38fc Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 Dec 2016 15:30:43 -0700 Subject: [PATCH 26/28] Add get_current() and enumerate(). --- Doc/library/_interpreters.rst | 10 ++++ Lib/test/test__interpreters.py | 88 ++++++++++++++++++++++++---------- Modules/_interpretersmodule.c | 27 ++++++++++- 3 files changed, 97 insertions(+), 28 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 445926ac4b87ba..8c58d22cca79df 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -31,6 +31,16 @@ support multiple interpreters. It defines the following functions: +.. function:: enumerate() + + Return a list of the IDs of every existing interpreter. + + +.. function:: get_current() + + Return the ID of the currently running interpreter. + + .. function:: create() Initialize a new Python interpreter and return its identifier. The diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 459ff8650a8470..c0836299b83117 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -67,7 +67,7 @@ def test_still_running_at_exit(self): class TestBase(unittest.TestCase): def tearDown(self): - for id in interpreters._enumerate(): + for id in interpreters.enumerate(): if id == 0: # main continue try: @@ -76,13 +76,49 @@ def tearDown(self): pass # already destroyed +class EnumerateTests(TestBase): + + def test_multiple(self): + main, = interpreters.enumerate() + id1 = interpreters.create() + id2 = interpreters.create() + ids = interpreters.enumerate() + + self.assertEqual(set(ids), {main, id1, id2}) + + def test_main_only(self): + main, = interpreters.enumerate() + + self.assertEqual(main, 0) + + +class GetCurrentTests(TestBase): + + def test_main(self): + main, = interpreters.enumerate() + id = interpreters.get_current() + + self.assertEqual(id, main) + + def test_sub(self): + id1 = interpreters.create() + ns = interpreters.run_string_unrestricted(id1, dedent(""" + import _interpreters + id = _interpreters.get_current() + """)) + id2 = ns['id'] + + self.assertEqual(id2, id1) + + class CreateTests(TestBase): def test_in_main(self): id = interpreters.create() - self.assertIn(id, interpreters._enumerate()) + self.assertIn(id, interpreters.enumerate()) + @unittest.skip('enable this test when working on pystate.c') def test_unique_id(self): seen = set() for _ in range(100): @@ -105,10 +141,10 @@ def f(): with lock: t.start() t.join() - self.assertIn(id, interpreters._enumerate()) + self.assertIn(id, interpreters.enumerate()) def test_in_subinterpreter(self): - main, = interpreters._enumerate() + main, = interpreters.enumerate() id1 = interpreters.create() ns = interpreters.run_string_unrestricted(id1, dedent(""" import _interpreters @@ -116,10 +152,10 @@ def test_in_subinterpreter(self): """)) id2 = ns['id'] - self.assertEqual(set(interpreters._enumerate()), {main, id1, id2}) + self.assertEqual(set(interpreters.enumerate()), {main, id1, id2}) def test_in_threaded_subinterpreter(self): - main, = interpreters._enumerate() + main, = interpreters.enumerate() id1 = interpreters.create() ns = None script = dedent(""" @@ -135,11 +171,11 @@ def f(): t.join() id2 = ns['id'] - self.assertEqual(set(interpreters._enumerate()), {main, id1, id2}) + self.assertEqual(set(interpreters.enumerate()), {main, id1, id2}) def test_after_destroy_all(self): - before = set(interpreters._enumerate()) + before = set(interpreters.enumerate()) # Create 3 subinterpreters. ids = [] for _ in range(3): @@ -150,10 +186,10 @@ def test_after_destroy_all(self): interpreters.destroy(id) # Finally, create another. id = interpreters.create() - self.assertEqual(set(interpreters._enumerate()), before | {id}) + self.assertEqual(set(interpreters.enumerate()), before | {id}) def test_after_destroy_some(self): - before = set(interpreters._enumerate()) + before = set(interpreters.enumerate()) # Create 3 subinterpreters. id1 = interpreters.create() id2 = interpreters.create() @@ -163,7 +199,7 @@ def test_after_destroy_some(self): interpreters.destroy(id3) # Finally, create another. id = interpreters.create() - self.assertEqual(set(interpreters._enumerate()), before | {id, id2}) + self.assertEqual(set(interpreters.enumerate()), before | {id, id2}) class DestroyTests(TestBase): @@ -172,25 +208,25 @@ def test_one(self): id1 = interpreters.create() id2 = interpreters.create() id3 = interpreters.create() - self.assertIn(id2, interpreters._enumerate()) + self.assertIn(id2, interpreters.enumerate()) interpreters.destroy(id2) - self.assertNotIn(id2, interpreters._enumerate()) - self.assertIn(id1, interpreters._enumerate()) - self.assertIn(id3, interpreters._enumerate()) + self.assertNotIn(id2, interpreters.enumerate()) + self.assertIn(id1, interpreters.enumerate()) + self.assertIn(id3, interpreters.enumerate()) def test_all(self): - before = set(interpreters._enumerate()) + before = set(interpreters.enumerate()) ids = set() for _ in range(3): id = interpreters.create() ids.add(id) - self.assertEqual(set(interpreters._enumerate()), before | ids) + self.assertEqual(set(interpreters.enumerate()), before | ids) for id in ids: interpreters.destroy(id) - self.assertEqual(set(interpreters._enumerate()), before) + self.assertEqual(set(interpreters.enumerate()), before) def test_main(self): - main, = interpreters._enumerate() + main, = interpreters.enumerate() with self.assertRaises(RuntimeError): interpreters.destroy(main) @@ -217,7 +253,7 @@ def test_bad_id(self): interpreters.destroy(-1) def test_from_current(self): - main, = interpreters._enumerate() + main, = interpreters.enumerate() id = interpreters.create() script = dedent(""" import _interpreters @@ -226,10 +262,10 @@ def test_from_current(self): with self.assertRaises(RuntimeError): interpreters.run_string(id, script) - self.assertEqual(set(interpreters._enumerate()), {main, id}) + self.assertEqual(set(interpreters.enumerate()), {main, id}) def test_from_sibling(self): - main, = interpreters._enumerate() + main, = interpreters.enumerate() id1 = interpreters.create() id2 = interpreters.create() script = dedent(""" @@ -238,7 +274,7 @@ def test_from_sibling(self): """).format(id2) interpreters.run_string(id1, script) - self.assertEqual(set(interpreters._enumerate()), {main, id1}) + self.assertEqual(set(interpreters.enumerate()), {main, id1}) def test_from_other_thread(self): id = interpreters.create() @@ -252,7 +288,7 @@ def f(): def test_still_running(self): # XXX Rewrite this test without files by using # run_string_unrestricted(). - main, = interpreters._enumerate() + main, = interpreters.enumerate() id = interpreters.create() def f(): interpreters.run_string(id, wait_script) @@ -265,7 +301,7 @@ def f(): interpreters.destroy(id) t.join() - self.assertEqual(set(interpreters._enumerate()), {main, id}) + self.assertEqual(set(interpreters.enumerate()), {main, id}) class RunStringTests(TestBase): @@ -389,7 +425,7 @@ def f(): def test_does_not_exist(self): id = 0 - while id in interpreters._enumerate(): + while id in interpreters.enumerate(): id += 1 with self.assertRaises(RuntimeError): interpreters.run_string(id, 'print("spam")') diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 299e71e620ddc3..a29333ac621df6 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -446,6 +446,27 @@ interp_enumerate(PyObject *self) return ids; } +PyDoc_STRVAR(enumerate_doc, +"enumerate() -> [ID]\n\ +\n\ +Return a list containing the ID of every existing interpreter."); + + +static PyObject * +interp_get_current(PyObject *self) +{ + PyInterpreterState *interp =_get_current(); + if (interp == NULL) + return NULL; + return _get_id(interp); +} + +PyDoc_STRVAR(get_current_doc, +"get_current() -> ID\n\ +\n\ +Return the ID of current interpreter."); + + static PyObject * interp_run_string(PyObject *self, PyObject *args) { @@ -518,8 +539,10 @@ static PyMethodDef module_functions[] = { {"destroy", (PyCFunction)interp_destroy, METH_VARARGS, destroy_doc}, - {"_enumerate", (PyCFunction)interp_enumerate, - METH_NOARGS, NULL}, + {"enumerate", (PyCFunction)interp_enumerate, + METH_NOARGS, enumerate_doc}, + {"get_current", (PyCFunction)interp_get_current, + METH_NOARGS, get_current_doc}, {"run_string", (PyCFunction)interp_run_string, METH_VARARGS, run_string_doc}, From c50c51c728d735fe32d661bc6c62a1f971e8f262 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 Dec 2016 15:33:17 -0700 Subject: [PATCH 27/28] Add is_running(). --- Doc/library/_interpreters.rst | 6 ++++++ Lib/test/test__interpreters.py | 30 ++++++++++++++++++++++++++++++ Modules/_interpretersmodule.c | 30 ++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 8c58d22cca79df..1bb107e500e4a5 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -41,6 +41,12 @@ It defines the following functions: Return the ID of the currently running interpreter. +.. function:: is_running(id) + + Return whether or not the identified interpreter is currently + running any code. + + .. function:: create() Initialize a new Python interpreter and return its identifier. The diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index c0836299b83117..49ed4444be84ee 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -111,6 +111,36 @@ def test_sub(self): self.assertEqual(id2, id1) +class IsRunningTests(TestBase): + + def test_main_running(self): + main, = interpreters.enumerate() + sub = interpreters.create() + main_running = interpreters.is_running(main) + sub_running = interpreters.is_running(sub) + + self.assertTrue(main_running) + self.assertFalse(sub_running) + + def test_sub_running(self): + main, = interpreters.enumerate() + sub1 = interpreters.create() + sub2 = interpreters.create() + ns = interpreters.run_string_unrestricted(sub1, dedent(f""" + import _interpreters + main = _interpreters.is_running({main}) + sub1 = _interpreters.is_running({sub1}) + sub2 = _interpreters.is_running({sub2}) + """)) + main_running = ns['main'] + sub1_running = ns['sub1'] + sub2_running = ns['sub2'] + + self.assertTrue(main_running) + self.assertTrue(sub1_running) + self.assertFalse(sub2_running) + + class CreateTests(TestBase): def test_in_main(self): diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index a29333ac621df6..375388fb75390f 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -533,6 +533,34 @@ Return True if the object's data may be shared between interpreters and\n\ False otherwise."); +static PyObject * +interp_is_running(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "is_running", 1, 1, &id)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "ID must be an int"); + return NULL; + } + + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + int is_running = _is_running(interp); + if (is_running < 0) + return NULL; + if (is_running) + Py_RETURN_TRUE; + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(is_running_doc, +"is_running(id) -> bool\n\ +\n\ +Return whether or not the identified interpreter is running."); + + static PyMethodDef module_functions[] = { {"create", (PyCFunction)interp_create, METH_VARARGS, create_doc}, @@ -543,6 +571,8 @@ static PyMethodDef module_functions[] = { METH_NOARGS, enumerate_doc}, {"get_current", (PyCFunction)interp_get_current, METH_NOARGS, get_current_doc}, + {"is_running", (PyCFunction)interp_is_running, + METH_VARARGS, is_running_doc}, {"run_string", (PyCFunction)interp_run_string, METH_VARARGS, run_string_doc}, From 271d20de5777c243043f13197b6d935946684d32 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 14:41:58 -0700 Subject: [PATCH 28/28] Add get_main(). --- Doc/library/_interpreters.rst | 5 +++++ Lib/test/test__interpreters.py | 10 ++++++++++ Modules/_interpretersmodule.c | 15 +++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 1bb107e500e4a5..bbc945016e2880 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -41,6 +41,11 @@ It defines the following functions: Return the ID of the currently running interpreter. +.. function:: get_main() + + Return the ID of the main interpreter. + + .. function:: is_running(id) Return whether or not the identified interpreter is currently diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 49ed4444be84ee..29d04aa4d00cfb 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -111,6 +111,16 @@ def test_sub(self): self.assertEqual(id2, id1) +class GetMainTests(TestBase): + + def test_main(self): + expected, = interpreters.enumerate() + main = interpreters.get_main() + + self.assertEqual(main, 0) + self.assertEqual(main, expected) + + class IsRunningTests(TestBase): def test_main_running(self): diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 375388fb75390f..f9b800cb1ace35 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -467,6 +467,19 @@ PyDoc_STRVAR(get_current_doc, Return the ID of current interpreter."); +static PyObject * +interp_get_main(PyObject *self) +{ + // Currently, 0 is always the main interpreter. + return PyLong_FromLongLong(0); +} + +PyDoc_STRVAR(get_main_doc, +"get_main() -> ID\n\ +\n\ +Return the ID of main interpreter."); + + static PyObject * interp_run_string(PyObject *self, PyObject *args) { @@ -571,6 +584,8 @@ static PyMethodDef module_functions[] = { METH_NOARGS, enumerate_doc}, {"get_current", (PyCFunction)interp_get_current, METH_NOARGS, get_current_doc}, + {"get_main", (PyCFunction)interp_get_main, + METH_NOARGS, get_main_doc}, {"is_running", (PyCFunction)interp_is_running, METH_VARARGS, is_running_doc},