From 7cc14ad04ab4dc2456851590f3240f234e348d43 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 14 Dec 2016 06:52:38 -0700 Subject: [PATCH 01/20] Add the _interpreters module to the stdlib. --- Doc/library/_interpreters.rst | 37 +++++++++++++++++++++++++++++ Lib/test/test__interpreters.py | 11 +++++++++ Modules/_interpretersmodule.c | 43 ++++++++++++++++++++++++++++++++++ setup.py | 3 +++ 4 files changed, 94 insertions(+) create mode 100644 Doc/library/_interpreters.rst create mode 100644 Lib/test/test__interpreters.py create mode 100644 Modules/_interpretersmodule.c diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst new file mode 100644 index 00000000000000..db0f2ef3942945 --- /dev/null +++ b/Doc/library/_interpreters.rst @@ -0,0 +1,37 @@ +:mod:`_interpreters` --- Low-level interpreters API +=================================================== + +.. module:: _interpreters + :synopsis: Low-level interpreters API. + +.. versionadded:: 3,7 + + :ref:`_sub-interpreter-support` + +threading + +-------------- + +This module provides low-level primitives for working with multiple +Python interpreters in the same process. + +.. XXX The :mod:`interpreters` module provides an easier to use and + higher-level API built on top of this module. + +This module is optional. It is provided by Python implementations which +support multiple interpreters. + +.. XXX For systems lacking the :mod:`_interpreters` module, the + :mod:`_dummy_interpreters` module is available. It duplicates this + module's interface and can be used as a drop-in replacement. + +It defines the following functions: + + +.. XXX TBD + + +**Caveats:** + +* ... + diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py new file mode 100644 index 00000000000000..157ef203b74907 --- /dev/null +++ b/Lib/test/test__interpreters.py @@ -0,0 +1,11 @@ +import unittest +from test import support +interpreters = support.import_module('_interpreters') + + +class InterpretersTests(unittest.TestCase): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c new file mode 100644 index 00000000000000..1103e1e37c48f1 --- /dev/null +++ b/Modules/_interpretersmodule.c @@ -0,0 +1,43 @@ + +/* interpreters module */ +/* low-level access to interpreter primitives */ + +#include "Python.h" + + +static PyMethodDef module_functions[] = { + {NULL, NULL} /* sentinel */ +}; + + +/* initialization function */ + +PyDoc_STRVAR(module_doc, +"This module provides primitive operations to manage Python interpreters.\n\ +The 'interpreters' module provides a more convenient interface."); + +static struct PyModuleDef interpretersmodule = { + PyModuleDef_HEAD_INIT, + "_interpreters", + module_doc, + -1, + module_functions, + NULL, + NULL, + NULL, + NULL +}; + + +PyMODINIT_FUNC +PyInit__interpreters(void) +{ + PyObject *module; + + module = PyModule_Create(&interpretersmodule); + if (module == NULL) + return NULL; + + + return module; +} diff --git a/setup.py b/setup.py index 6a05643838bb4e..d95015721e008b 100644 --- a/setup.py +++ b/setup.py @@ -694,6 +694,9 @@ def detect_modules(self): # syslog daemon interface exts.append( Extension('syslog', ['syslogmodule.c']) ) + # Python interface to subinterpreter C-API. + exts.append(Extension('_interpreters', ['_interpretersmodule.c'])) + # # Here ends the simple stuff. From here on, modules need certain # libraries, are platform-specific, or present other surprises. From 84920ad6558c7fad4ba84b9f6eb7c330f95f8560 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 Dec 2016 15:28:53 -0700 Subject: [PATCH 02/20] Add create() and destroy(). --- Doc/library/_interpreters.rst | 13 ++- Lib/test/test__interpreters.py | 157 ++++++++++++++++++++++++++++++- Modules/_interpretersmodule.c | 167 ++++++++++++++++++++++++++++++++- 3 files changed, 334 insertions(+), 3 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index db0f2ef3942945..259b3ff67f13c3 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -28,7 +28,18 @@ support multiple interpreters. It defines the following functions: -.. XXX TBD +.. function:: create() + + Initialize a new Python interpreter and return its identifier. The + interpreter will be created in the current thread and will remain + idle until something is run in it. + + +.. function:: destroy(id) + + Finalize and destroy the identified interpreter. + +.. XXX must not be running? **Caveats:** diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 157ef203b74907..dd84db647bf1d5 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -1,10 +1,165 @@ +import threading import unittest + from test import support interpreters = support.import_module('_interpreters') class InterpretersTests(unittest.TestCase): - pass + + def setUp(self): + self.ids = [] + self.lock = threading.Lock() + + def tearDown(self): + for id in self.ids: + try: + interpreters.destroy(id) + except RuntimeError: + pass # already destroyed + + def _create(self): + id = interpreters.create() + self.ids.append(id) + return id + + def test_create_in_main(self): + id = interpreters.create() + self.ids.append(id) + + self.assertIn(id, interpreters._enumerate()) + + def test_create_unique_id(self): + seen = set() + for _ in range(100): + id = self._create() + interpreters.destroy(id) + seen.add(id) + + self.assertEqual(len(seen), 100) + + def test_create_in_thread(self): + id = None + def f(): + nonlocal id + id = interpreters.create() + self.ids.append(id) + self.lock.acquire() + self.lock.release() + + t = threading.Thread(target=f) + with self.lock: + t.start() + t.join() + self.assertIn(id, interpreters._enumerate()) + + @unittest.skip('waiting for run_string()') + def test_create_in_subinterpreter(self): + raise NotImplementedError + + @unittest.skip('waiting for run_string()') + def test_create_in_threaded_subinterpreter(self): + raise NotImplementedError + + def test_create_after_destroy_all(self): + before = set(interpreters._enumerate()) + # Create 3 subinterpreters. + ids = [] + for _ in range(3): + id = interpreters.create() + ids.append(id) + # Now destroy them. + for id in ids: + interpreters.destroy(id) + # Finally, create another. + id = interpreters.create() + self.ids.append(id) + self.assertEqual(set(interpreters._enumerate()), before | {id}) + + def test_create_after_destroy_some(self): + before = set(interpreters._enumerate()) + # Create 3 subinterpreters. + id1 = interpreters.create() + id2 = interpreters.create() + self.ids.append(id2) + id3 = interpreters.create() + # Now destroy 2 of them. + interpreters.destroy(id1) + interpreters.destroy(id3) + # Finally, create another. + id = interpreters.create() + self.ids.append(id) + self.assertEqual(set(interpreters._enumerate()), before | {id, id2}) + + def test_destroy_one(self): + id1 = self._create() + id2 = self._create() + id3 = self._create() + self.assertIn(id2, interpreters._enumerate()) + interpreters.destroy(id2) + self.assertNotIn(id2, interpreters._enumerate()) + self.assertIn(id1, interpreters._enumerate()) + self.assertIn(id3, interpreters._enumerate()) + + def test_destroy_all(self): + before = set(interpreters._enumerate()) + ids = set() + for _ in range(3): + id = self._create() + ids.add(id) + self.assertEqual(set(interpreters._enumerate()), before | ids) + for id in ids: + interpreters.destroy(id) + self.assertEqual(set(interpreters._enumerate()), before) + + def test_destroy_main(self): + main, = interpreters._enumerate() + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + def f(): + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_destroy_already_destroyed(self): + id = interpreters.create() + interpreters.destroy(id) + with self.assertRaises(RuntimeError): + interpreters.destroy(id) + + def test_destroy_does_not_exist(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(1_000_000) + + def test_destroy_bad_id(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(-1) + + @unittest.skip('waiting for run_string()') + def test_destroy_from_current(self): + raise NotImplementedError + + @unittest.skip('waiting for run_string()') + def test_destroy_from_sibling(self): + raise NotImplementedError + + def test_destroy_from_other_thread(self): + id = interpreters.create() + self.ids.append(id) + def f(): + interpreters.destroy(id) + + t = threading.Thread(target=f) + t.start() + t.join() + + @unittest.skip('waiting for run_string()') + def test_destroy_still_running(self): + raise NotImplementedError if __name__ == "__main__": diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 1103e1e37c48f1..97235034e722e7 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -5,8 +5,173 @@ #include "Python.h" +static PyObject * +_get_id(PyInterpreterState *interp) +{ + unsigned long id = PyInterpreterState_GetID(interp); + if (id == 0 && PyErr_Occurred() != NULL) + return NULL; + return PyLong_FromUnsignedLong(id); +} + +static PyInterpreterState * +_look_up(PyObject *requested_id) +{ + PyObject * id; + PyInterpreterState *interp; + + interp = PyInterpreterState_Head(); + while (interp != NULL) { + id = _get_id(interp); + if (id == NULL) + return NULL; + if (requested_id == id) + return interp; + interp = PyInterpreterState_Next(interp); + } + + PyErr_Format(PyExc_RuntimeError, + "unrecognized interpreter ID %R", requested_id); + return NULL; +} + +static PyObject * +_get_current(void) +{ + PyThreadState *tstate; + PyInterpreterState *interp; + + tstate = PyThreadState_Get(); + if (tstate == NULL) + return NULL; + interp = tstate->interp; + + // get ID + return _get_id(interp); +} + + +/* module level code ********************************************************/ + +// XXX track count? + +static PyObject * +interp_create(PyObject *self, PyObject *args) +{ + if (!PyArg_UnpackTuple(args, "create", 0, 0)) + return NULL; + + // Create and initialize the new interpreter. + PyThreadState *tstate, *save_tstate; + save_tstate = PyThreadState_Swap(NULL); + tstate = Py_NewInterpreter(); + PyThreadState_Swap(save_tstate); + if (tstate == NULL) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); + return NULL; + } + return _get_id(tstate->interp); +} + +PyDoc_STRVAR(create_doc, +"create() -> ID\n\ +\n\ +Create a new interpreter and return a unique generated ID."); + + +static PyObject * +interp_destroy(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "destroy", 1, 1, &id)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "ID must be an int"); + return NULL; + } + + // Ensure the ID is not the current interpreter. + PyObject *current = _get_current(); + if (current == NULL) + return NULL; + if (PyObject_RichCompareBool(id, current, Py_EQ) != 0) { + PyErr_SetString(PyExc_RuntimeError, + "cannot destroy the current interpreter"); + return NULL; + } + + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + + // Destroy the interpreter. + //PyInterpreterState_Delete(interp); + PyThreadState *tstate, *save_tstate; + tstate = PyInterpreterState_ThreadHead(interp); // XXX Is this the right one? + save_tstate = PyThreadState_Swap(tstate); + // XXX Stop current execution? + Py_EndInterpreter(tstate); // XXX Handle possible errors? + PyThreadState_Swap(save_tstate); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(destroy_doc, +"destroy(ID)\n\ +\n\ +Destroy the identified interpreter.\n\ +\n\ +Attempting to destroy the current interpreter results in a RuntimeError.\n\ +So does an unrecognized ID."); + + +static PyObject * +interp_enumerate(PyObject *self) +{ + PyObject *ids, *id; + PyInterpreterState *interp; + + // XXX Handle multiple main interpreters. + + ids = PyList_New(0); + if (ids == NULL) + return NULL; + + interp = PyInterpreterState_Head(); + while (interp != NULL) { + id = _get_id(interp); + if (id == NULL) + return NULL; + // insert at front of list + if (PyList_Insert(ids, 0, id) < 0) + return NULL; + + interp = PyInterpreterState_Next(interp); + } + + return ids; +} + +PyDoc_STRVAR(enumerate_doc, +"enumerate() -> [ID]\n\ +\n\ +Return a list containing the ID of every existing interpreter."); + + static PyMethodDef module_functions[] = { - {NULL, NULL} /* sentinel */ + {"create", (PyCFunction)interp_create, + METH_VARARGS, create_doc}, + {"destroy", (PyCFunction)interp_destroy, + METH_VARARGS, destroy_doc}, + + {"_enumerate", (PyCFunction)interp_enumerate, + METH_NOARGS, enumerate_doc}, + + {NULL, NULL} /* sentinel */ }; From 89f16eeba6180ea59df5335ec2fa6e1d4f5fbe7c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 31 Dec 2016 18:49:38 -0700 Subject: [PATCH 03/20] Finish nearly all the create/destroy tests. --- Lib/test/test__interpreters.py | 162 +++++++++++++++++++--------- Modules/_interpretersmodule.c | 186 +++++++++++++++++++++++++++------ 2 files changed, 265 insertions(+), 83 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index dd84db647bf1d5..c46ca0236b96d2 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -1,67 +1,108 @@ +import contextlib +import os import threading import unittest from test import support + interpreters = support.import_module('_interpreters') -class InterpretersTests(unittest.TestCase): +@contextlib.contextmanager +def _blocked(): + r, w = os.pipe() + wait_script = """if True: + import select + # Wait for a "done" signal. + select.select([{}], [], []) + + #import time + #time.sleep(1_000_000) + """.format(r) + try: + yield wait_script + finally: + os.write(w, b'') # release! + os.close(r) + os.close(w) + - def setUp(self): - self.ids = [] - self.lock = threading.Lock() +class TestBase(unittest.TestCase): def tearDown(self): - for id in self.ids: + for id in interpreters._enumerate(): + if id == 0: # main + continue try: interpreters.destroy(id) except RuntimeError: pass # already destroyed - def _create(self): - id = interpreters.create() - self.ids.append(id) - return id - def test_create_in_main(self): +class CreateTests(TestBase): + + def test_in_main(self): id = interpreters.create() - self.ids.append(id) self.assertIn(id, interpreters._enumerate()) - def test_create_unique_id(self): + def test_unique_id(self): seen = set() for _ in range(100): - id = self._create() + id = interpreters.create() interpreters.destroy(id) seen.add(id) self.assertEqual(len(seen), 100) - def test_create_in_thread(self): + def test_in_thread(self): + lock = threading.Lock() id = None def f(): nonlocal id id = interpreters.create() - self.ids.append(id) - self.lock.acquire() - self.lock.release() + lock.acquire() + lock.release() t = threading.Thread(target=f) - with self.lock: + with lock: t.start() t.join() self.assertIn(id, interpreters._enumerate()) - @unittest.skip('waiting for run_string()') - def test_create_in_subinterpreter(self): - raise NotImplementedError + def test_in_subinterpreter(self): + main, = interpreters._enumerate() + id = interpreters.create() + interpreters._run_string(id, """if True: + import _interpreters + id = _interpreters.create() + #_interpreters.create() + """) + + ids = interpreters._enumerate() + self.assertIn(id, ids) + self.assertIn(main, ids) + self.assertEqual(len(ids), 3) + + def test_in_threaded_subinterpreter(self): + main, = interpreters._enumerate() + id = interpreters.create() + def f(): + interpreters._run_string(id, """if True: + import _interpreters + _interpreters.create() + """) + + t = threading.Thread(target=f) + t.start() + t.join() - @unittest.skip('waiting for run_string()') - def test_create_in_threaded_subinterpreter(self): - raise NotImplementedError + ids = interpreters._enumerate() + self.assertIn(id, ids) + self.assertIn(main, ids) + self.assertEqual(len(ids), 3) - def test_create_after_destroy_all(self): + def test_after_destroy_all(self): before = set(interpreters._enumerate()) # Create 3 subinterpreters. ids = [] @@ -73,46 +114,46 @@ def test_create_after_destroy_all(self): interpreters.destroy(id) # Finally, create another. id = interpreters.create() - self.ids.append(id) self.assertEqual(set(interpreters._enumerate()), before | {id}) - def test_create_after_destroy_some(self): + def test_after_destroy_some(self): before = set(interpreters._enumerate()) # Create 3 subinterpreters. id1 = interpreters.create() id2 = interpreters.create() - self.ids.append(id2) id3 = interpreters.create() # Now destroy 2 of them. interpreters.destroy(id1) interpreters.destroy(id3) # Finally, create another. id = interpreters.create() - self.ids.append(id) self.assertEqual(set(interpreters._enumerate()), before | {id, id2}) - def test_destroy_one(self): - id1 = self._create() - id2 = self._create() - id3 = self._create() + +class DestroyTests(TestBase): + + def test_one(self): + id1 = interpreters.create() + id2 = interpreters.create() + id3 = interpreters.create() self.assertIn(id2, interpreters._enumerate()) interpreters.destroy(id2) self.assertNotIn(id2, interpreters._enumerate()) self.assertIn(id1, interpreters._enumerate()) self.assertIn(id3, interpreters._enumerate()) - def test_destroy_all(self): + def test_all(self): before = set(interpreters._enumerate()) ids = set() for _ in range(3): - id = self._create() + id = interpreters.create() ids.add(id) self.assertEqual(set(interpreters._enumerate()), before | ids) for id in ids: interpreters.destroy(id) self.assertEqual(set(interpreters._enumerate()), before) - def test_destroy_main(self): + def test_main(self): main, = interpreters._enumerate() with self.assertRaises(RuntimeError): interpreters.destroy(main) @@ -125,31 +166,40 @@ def f(): t.start() t.join() - def test_destroy_already_destroyed(self): + def test_already_destroyed(self): id = interpreters.create() interpreters.destroy(id) with self.assertRaises(RuntimeError): interpreters.destroy(id) - def test_destroy_does_not_exist(self): + def test_does_not_exist(self): with self.assertRaises(RuntimeError): interpreters.destroy(1_000_000) - def test_destroy_bad_id(self): + def test_bad_id(self): with self.assertRaises(RuntimeError): interpreters.destroy(-1) - @unittest.skip('waiting for run_string()') - def test_destroy_from_current(self): - raise NotImplementedError + def test_from_current(self): + id = interpreters.create() + with self.assertRaises(RuntimeError): + interpreters._run_string(id, """if True: + import _interpreters + _interpreters.destroy({}) + """.format(id)) - @unittest.skip('waiting for run_string()') - def test_destroy_from_sibling(self): - raise NotImplementedError + def test_from_sibling(self): + main, = interpreters._enumerate() + id1 = interpreters.create() + id2 = interpreters.create() + interpreters._run_string(id1, """if True: + import _interpreters + _interpreters.destroy({}) + """.format(id2)) + self.assertEqual(set(interpreters._enumerate()), {main, id1}) - def test_destroy_from_other_thread(self): + def test_from_other_thread(self): id = interpreters.create() - self.ids.append(id) def f(): interpreters.destroy(id) @@ -157,9 +207,21 @@ def f(): t.start() t.join() - @unittest.skip('waiting for run_string()') - def test_destroy_still_running(self): - raise NotImplementedError + @unittest.skip('not working yet') + def test_still_running(self): + main, = interpreters._enumerate() + id = interpreters.create() + def f(): + interpreters._run_string(id, wait_script) + + t = threading.Thread(target=f) + with _blocked() as wait_script: + t.start() + with self.assertRaises(RuntimeError): + interpreters.destroy(id) + + t.join() + self.assertEqual(set(interpreters._enumerate()), {main, id}) if __name__ == "__main__": diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 97235034e722e7..ce44df269bacd2 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -3,53 +3,131 @@ /* low-level access to interpreter primitives */ #include "Python.h" +#include "frameobject.h" -static PyObject * -_get_id(PyInterpreterState *interp) +static PyInterpreterState * +_get_current(void) { - unsigned long id = PyInterpreterState_GetID(interp); - if (id == 0 && PyErr_Occurred() != NULL) + PyThreadState *tstate; + + tstate = PyThreadState_Get(); + if (tstate == NULL) return NULL; - return PyLong_FromUnsignedLong(id); + return tstate->interp; } static PyInterpreterState * -_look_up(PyObject *requested_id) +_look_up_int64(PY_INT64_T requested_id) { - PyObject * id; - PyInterpreterState *interp; + if (requested_id < 0) + goto error; - interp = PyInterpreterState_Head(); + PyInterpreterState *interp = PyInterpreterState_Head(); while (interp != NULL) { - id = _get_id(interp); - if (id == NULL) + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) return NULL; if (requested_id == id) return interp; interp = PyInterpreterState_Next(interp); } +error: PyErr_Format(PyExc_RuntimeError, - "unrecognized interpreter ID %R", requested_id); + "unrecognized interpreter ID %lld", requested_id); return NULL; } -static PyObject * -_get_current(void) +static PyInterpreterState * +_look_up(PyObject *requested_id) { - PyThreadState *tstate; - PyInterpreterState *interp; + long long id = PyLong_AsLongLong(requested_id); + if (id == -1 && PyErr_Occurred() != NULL) + return NULL; + // XXX Fail if larger than INT64_MAX? + return _look_up_int64(id); +} - tstate = PyThreadState_Get(); - if (tstate == NULL) +static PyObject * +_get_id(PyInterpreterState *interp) +{ + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) return NULL; - interp = tstate->interp; + return PyLong_FromLongLong(id); +} - // get ID - return _get_id(interp); +static int +_is_running(PyInterpreterState *interp) +{ + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + if (PyThreadState_Next(tstate) != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "interpreter has more than one thread"); + return -1; + } + PyFrameObject *frame = _PyThreadState_GetFrame(tstate); + if (frame == NULL) { + if (PyErr_Occurred() != NULL) + return -1; + return 0; + } + return (int)(frame->f_executing); } +static int +_ensure_not_running(PyInterpreterState *interp) +{ + int is_running = _is_running(interp); + if (is_running < 0) + return -1; + if (is_running) { + PyErr_Format(PyExc_RuntimeError, "interpreter already running"); + return -1; + } + return 0; +} + +static int +_run_string(PyInterpreterState *interp, const char *codestr) +{ + PyObject *result = NULL; + PyObject *exc = NULL, *value = NULL, *tb = NULL; + + if (_ensure_not_running(interp) < 0) + return -1; + + // Switch to interpreter. + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + PyThreadState *save_tstate = PyThreadState_Swap(tstate); + + // Run the string (see PyRun_SimpleStringFlags). + // XXX How to handle sys.exit()? + PyObject *m = PyImport_AddModule("__main__"); + if (m == NULL) { + PyErr_Fetch(&exc, &value, &tb); + goto done; + } + PyObject *d = PyModule_GetDict(m); + result = PyRun_StringFlags(codestr, Py_file_input, d, d, NULL); + if (result == NULL) { + // Get the exception from the subinterpreter. + PyErr_Fetch(&exc, &value, &tb); + goto done; + } + Py_DECREF(result); // We throw away the result. + +done: + // Switch back. + if (save_tstate != NULL) + PyThreadState_Swap(save_tstate); + + // Propagate any exception out to the caller. + PyErr_Restore(exc, value, tb); + + return (result == NULL) ? -1 : 0; +} /* module level code ********************************************************/ @@ -93,19 +171,25 @@ interp_destroy(PyObject *self, PyObject *args) return NULL; } - // Ensure the ID is not the current interpreter. - PyObject *current = _get_current(); + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + + // Ensure we don't try to destroy the current interpreter. + PyInterpreterState *current = _get_current(); if (current == NULL) return NULL; - if (PyObject_RichCompareBool(id, current, Py_EQ) != 0) { + if (interp == current) { PyErr_SetString(PyExc_RuntimeError, "cannot destroy the current interpreter"); return NULL; } - // Look up the interpreter. - PyInterpreterState *interp = _look_up(id); - if (interp == NULL) + // Ensure the interpreter isn't running. + /* XXX We *could* support destroying a running interpreter but + aren't going to worry about it for now. */ + if (_ensure_not_running(interp) < 0) return NULL; // Destroy the interpreter. @@ -156,11 +240,44 @@ interp_enumerate(PyObject *self) return ids; } -PyDoc_STRVAR(enumerate_doc, -"enumerate() -> [ID]\n\ -\n\ -Return a list containing the ID of every existing interpreter."); +static PyObject * +interp_run_string(PyObject *self, PyObject *args) +{ + PyObject *id, *code; + if (!PyArg_UnpackTuple(args, "run_string", 2, 2, &id, &code)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int"); + return NULL; + } + if (!PyUnicode_Check(code)) { + PyErr_SetString(PyExc_TypeError, + "second arg (code) must be a string"); + return NULL; + } + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + + // Extract code. + Py_ssize_t size; + const char *codestr = PyUnicode_AsUTF8AndSize(code, &size); + if (codestr == NULL) + return NULL; + if (strlen(codestr) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, + "source code string cannot contain null bytes"); + return NULL; + } + + // Run the code in the interpreter. + if (_run_string(interp, codestr) < 0) + return NULL; + else + Py_RETURN_NONE; +} static PyMethodDef module_functions[] = { {"create", (PyCFunction)interp_create, @@ -168,8 +285,11 @@ static PyMethodDef module_functions[] = { {"destroy", (PyCFunction)interp_destroy, METH_VARARGS, destroy_doc}, - {"_enumerate", (PyCFunction)interp_enumerate, - METH_NOARGS, enumerate_doc}, + {"_enumerate", (PyCFunction)interp_enumerate, + METH_NOARGS, NULL}, + + {"_run_string", (PyCFunction)interp_run_string, + METH_VARARGS, NULL}, {NULL, NULL} /* sentinel */ }; From 189a2fd27d5de6d28752d798a681d37ee4a07e33 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 Dec 2016 15:32:18 -0700 Subject: [PATCH 04/20] Add run_string(). --- Doc/library/_interpreters.rst | 18 +++- Lib/test/test__interpreters.py | 190 ++++++++++++++++++++++++++++++--- Modules/_interpretersmodule.c | 11 +- 3 files changed, 203 insertions(+), 16 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 259b3ff67f13c3..8d05622f5af106 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -27,7 +27,6 @@ support multiple interpreters. It defines the following functions: - .. function:: create() Initialize a new Python interpreter and return its identifier. The @@ -42,6 +41,23 @@ It defines the following functions: .. XXX must not be running? +.. function:: run_string(id, command) + + A wrapper around :c:func:`PyRun_SimpleString` which runs the provided + Python program using the identified interpreter. Providing an + invalid or unknown ID results in a RuntimeError, likewise if the main + interpreter or any other running interpreter is used. + + Any value returned from the code is thrown away, similar to what + threads do. If the code results in an exception then that exception + is raised in the thread in which run_string() was called, similar to + how :func:`exec` works. This aligns with how interpreters are not + inherently threaded. + +.. XXX must not be running already? +.. XXX sys.exit() (and SystemExit) is swallowed? + + **Caveats:** * ... diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index c46ca0236b96d2..f6392ae7abaeab 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -1,5 +1,9 @@ import contextlib import os +import os.path +import shutil +import tempfile +from textwrap import dedent import threading import unittest @@ -11,14 +15,14 @@ @contextlib.contextmanager def _blocked(): r, w = os.pipe() - wait_script = """if True: + wait_script = dedent(""" import select # Wait for a "done" signal. select.select([{}], [], []) #import time #time.sleep(1_000_000) - """.format(r) + """).format(r) try: yield wait_script finally: @@ -73,11 +77,10 @@ def f(): def test_in_subinterpreter(self): main, = interpreters._enumerate() id = interpreters.create() - interpreters._run_string(id, """if True: + interpreters.run_string(id, dedent(""" import _interpreters id = _interpreters.create() - #_interpreters.create() - """) + """)) ids = interpreters._enumerate() self.assertIn(id, ids) @@ -88,10 +91,10 @@ def test_in_threaded_subinterpreter(self): main, = interpreters._enumerate() id = interpreters.create() def f(): - interpreters._run_string(id, """if True: + interpreters.run_string(id, dedent(""" import _interpreters _interpreters.create() - """) + """)) t = threading.Thread(target=f) t.start() @@ -102,6 +105,7 @@ def f(): self.assertIn(main, ids) self.assertEqual(len(ids), 3) + def test_after_destroy_all(self): before = set(interpreters._enumerate()) # Create 3 subinterpreters. @@ -183,19 +187,19 @@ def test_bad_id(self): def test_from_current(self): id = interpreters.create() with self.assertRaises(RuntimeError): - interpreters._run_string(id, """if True: + interpreters.run_string(id, dedent(""" import _interpreters _interpreters.destroy({}) - """.format(id)) + """).format(id)) def test_from_sibling(self): main, = interpreters._enumerate() id1 = interpreters.create() id2 = interpreters.create() - interpreters._run_string(id1, """if True: + interpreters.run_string(id1, dedent(""" import _interpreters _interpreters.destroy({}) - """.format(id2)) + """).format(id2)) self.assertEqual(set(interpreters._enumerate()), {main, id1}) def test_from_other_thread(self): @@ -212,7 +216,7 @@ def test_still_running(self): main, = interpreters._enumerate() id = interpreters.create() def f(): - interpreters._run_string(id, wait_script) + interpreters.run_string(id, wait_script) t = threading.Thread(target=f) with _blocked() as wait_script: @@ -224,5 +228,165 @@ def f(): self.assertEqual(set(interpreters._enumerate()), {main, id}) -if __name__ == "__main__": +class RunStringTests(TestBase): + + SCRIPT = dedent(""" + with open('{}', 'w') as out: + out.write('{}') + """) + FILENAME = 'spam' + + def setUp(self): + self.id = interpreters.create() + self.dirname = None + self.filename = None + + def tearDown(self): + if self.dirname is not None: + shutil.rmtree(self.dirname) + super().tearDown() + + def _resolve_filename(self, name=None): + if name is None: + name = self.FILENAME + if self.dirname is None: + self.dirname = tempfile.mkdtemp() + return os.path.join(self.dirname, name) + + def _empty_file(self): + self.filename = self._resolve_filename() + support.create_empty_file(self.filename) + return self.filename + + def assert_file_contains(self, expected, filename=None): + if filename is None: + filename = self.filename + self.assertIsNot(filename, None) + with open(filename) as out: + content = out.read() + self.assertEqual(content, expected) + + def test_success(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = self.SCRIPT.format(filename, expected) + interpreters.run_string(self.id, script) + + self.assert_file_contains(expected) + + def test_in_thread(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = self.SCRIPT.format(filename, expected) + def f(): + interpreters.run_string(self.id, script) + + t = threading.Thread(target=f) + t.start() + t.join() + + self.assert_file_contains(expected) + + def test_create_thread(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = dedent(""" + import threading + def f(): + with open('{}', 'w') as out: + out.write('{}') + + t = threading.Thread(target=f) + t.start() + t.join() + """).format(filename, expected) + interpreters.run_string(self.id, script) + + self.assert_file_contains(expected) + + @unittest.skip('not working yet') + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + def test_fork(self): + filename = self._empty_file() + expected = 'spam spam spam spam spam' + script = dedent(""" + import os + import sys + pid = os.fork() + if pid == 0: + with open('{}', 'w') as out: + out.write('{}') + sys.exit(0) + """).format(filename, expected) + interpreters.run_string(self.id, script) + + self.assert_file_contains(expected) + + @unittest.skip('not working yet') + def test_already_running(self): + def f(): + interpreters.run_string(self.id, wait_script) + + t = threading.Thread(target=f) + with _blocked() as wait_script: + t.start() + with self.assertRaises(RuntimeError): + interpreters.run_string(self.id, 'print("spam")') + t.join() + + def test_does_not_exist(self): + id = 0 + while id in interpreters._enumerate(): + id += 1 + with self.assertRaises(RuntimeError): + interpreters.run_string(id, 'print("spam")') + + def test_error_id(self): + with self.assertRaises(RuntimeError): + interpreters.run_string(-1, 'print("spam")') + + def test_bad_id(self): + with self.assertRaises(TypeError): + interpreters.run_string('spam', 'print("spam")') + + def test_bad_code(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, 10) + + def test_bytes_for_code(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, b'print("spam")') + + def test_invalid_syntax(self): + with self.assertRaises(SyntaxError): + # missing close paren + interpreters.run_string(self.id, 'print("spam"') + + def test_failure(self): + with self.assertRaises(Exception) as caught: + interpreters.run_string(self.id, 'raise Exception("spam")') + self.assertEqual(str(caught.exception), 'spam') + + def test_sys_exit(self): + with self.assertRaises(SystemExit) as cm: + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit() + """)) + self.assertIsNone(cm.exception.code) + + with self.assertRaises(SystemExit) as cm: + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit(42) + """)) + self.assertEqual(cm.exception.code, 42) + + def test_SystemError(self): + with self.assertRaises(SystemExit) as cm: + interpreters.run_string(self.id, 'raise SystemExit(42)') + self.assertEqual(cm.exception.code, 42) + + +if __name__ == '__main__': unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index ce44df269bacd2..a46610e5a10899 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -279,6 +279,13 @@ interp_run_string(PyObject *self, PyObject *args) Py_RETURN_NONE; } +PyDoc_STRVAR(run_string_doc, +"run_string(ID, sourcetext) -> run_id\n\ +\n\ +Execute the provided string in the identified interpreter.\n\ +See PyRun_SimpleStrings."); + + static PyMethodDef module_functions[] = { {"create", (PyCFunction)interp_create, METH_VARARGS, create_doc}, @@ -288,8 +295,8 @@ static PyMethodDef module_functions[] = { {"_enumerate", (PyCFunction)interp_enumerate, METH_NOARGS, NULL}, - {"_run_string", (PyCFunction)interp_run_string, - METH_VARARGS, NULL}, + {"run_string", (PyCFunction)interp_run_string, + METH_VARARGS, run_string_doc}, {NULL, NULL} /* sentinel */ }; From e144c965fbdd824ce204bd6b7f03a6ff3099148c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Jan 2017 15:42:08 -0700 Subject: [PATCH 05/20] Get tricky tests working. --- Lib/test/test__interpreters.py | 53 +++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index f6392ae7abaeab..a23246ea2a3f3a 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -13,22 +13,18 @@ @contextlib.contextmanager -def _blocked(): - r, w = os.pipe() +def _blocked(dirname): + filename = os.path.join(dirname, '.lock') wait_script = dedent(""" - import select - # Wait for a "done" signal. - select.select([{}], [], []) - - #import time - #time.sleep(1_000_000) - """).format(r) + import os.path + import time + while not os.path.exists('{}'): + time.sleep(0.1) + """).format(filename) try: yield wait_script finally: - os.write(w, b'') # release! - os.close(r) - os.close(w) + support.create_empty_file(filename) class TestBase(unittest.TestCase): @@ -211,15 +207,15 @@ def f(): t.start() t.join() - @unittest.skip('not working yet') def test_still_running(self): main, = interpreters._enumerate() id = interpreters.create() def f(): interpreters.run_string(id, wait_script) + dirname = tempfile.mkdtemp() t = threading.Thread(target=f) - with _blocked() as wait_script: + with _blocked(dirname) as wait_script: t.start() with self.assertRaises(RuntimeError): interpreters.destroy(id) @@ -243,7 +239,10 @@ def setUp(self): def tearDown(self): if self.dirname is not None: - shutil.rmtree(self.dirname) + try: + shutil.rmtree(self.dirname) + except FileNotFoundError: + pass # already deleted super().tearDown() def _resolve_filename(self, name=None): @@ -304,31 +303,39 @@ def f(): self.assert_file_contains(expected) - @unittest.skip('not working yet') @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") def test_fork(self): filename = self._empty_file() expected = 'spam spam spam spam spam' script = dedent(""" import os - import sys + r, w = os.pipe() pid = os.fork() - if pid == 0: - with open('{}', 'w') as out: + if pid == 0: # child + import sys + filename = '{}' + with open(filename, 'w') as out: out.write('{}') - sys.exit(0) + os.write(w, b'done!') + else: + import select + try: + select.select([r], [], []) + finally: + os.close(r) + os.close(w) """).format(filename, expected) + # XXX Kill the child process in a unittest-friendly way. interpreters.run_string(self.id, script) - self.assert_file_contains(expected) - @unittest.skip('not working yet') def test_already_running(self): def f(): interpreters.run_string(self.id, wait_script) t = threading.Thread(target=f) - with _blocked() as wait_script: + dirname = tempfile.mkdtemp() + with _blocked(dirname) as wait_script: t.start() with self.assertRaises(RuntimeError): interpreters.run_string(self.id, 'print("spam")') From f3c71946a8c07cc10781d5054184a2d57f8cbf8d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 2 Jan 2017 15:46:07 -0700 Subject: [PATCH 06/20] Add a test for a still running interpreter when main exits. --- Lib/test/test__interpreters.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index a23246ea2a3f3a..24724b662dd400 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -8,10 +8,22 @@ import unittest from test import support +from test.support import script_helper interpreters = support.import_module('_interpreters') +SCRIPT_THREADED_INTERP = """\ +import threading +import _interpreters +def f(): + _interpreters.run_string(id, {!r}) + +t = threading.Thread(target=f) +t.start() +""" + + @contextlib.contextmanager def _blocked(dirname): filename = os.path.join(dirname, '.lock') @@ -27,6 +39,27 @@ def _blocked(dirname): support.create_empty_file(filename) +class InterpreterTests(unittest.TestCase): + + def setUp(self): + self.dirname = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.dirname) + + def test_still_running_at_exit(self): + script = SCRIPT_THREADED_INTERP.format("""if True: + import time + # Give plenty of time for the main interpreter to finish. + time.sleep(1_000_000) + """) + filename = script_helper.make_script(self.dirname, 'interp', script) + proc = script_helper.spawn_python(filename) + retcode = proc.wait() + + self.assertEqual(retcode, 0) + + class TestBase(unittest.TestCase): def tearDown(self): From 2b7b6c7d7da6b3caead86ff14db7c42ac203ffe8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jan 2017 17:48:34 -0700 Subject: [PATCH 07/20] Add run_string_unrestricted(). --- Doc/library/_interpreters.rst | 12 +++- Lib/test/test__interpreters.py | 101 ++++++++++++++++++++++++++------- Modules/_interpretersmodule.c | 95 +++++++++++++++++++++++++------ 3 files changed, 168 insertions(+), 40 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 8d05622f5af106..a4f86e38496e5b 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -38,8 +38,6 @@ It defines the following functions: Finalize and destroy the identified interpreter. -.. XXX must not be running? - .. function:: run_string(id, command) @@ -54,10 +52,18 @@ It defines the following functions: how :func:`exec` works. This aligns with how interpreters are not inherently threaded. -.. XXX must not be running already? .. XXX sys.exit() (and SystemExit) is swallowed? +.. function:: run_string_unrestricted(id, command, ns=None) + + Like :c:func:`run_string` but returns the dict in which the code + was executed. It also supports providing a namespace that gets + merged into the execution namespace before execution. Note that + this allows objects to leak between interpreters, which may not + be desirable. + + **Caveats:** * ... diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 24724b662dd400..edd279c16c4935 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -105,34 +105,33 @@ def f(): def test_in_subinterpreter(self): main, = interpreters._enumerate() - id = interpreters.create() - interpreters.run_string(id, dedent(""" + id1 = interpreters.create() + ns = interpreters.run_string_unrestricted(id1, dedent(""" import _interpreters id = _interpreters.create() """)) + id2 = ns['id'] - ids = interpreters._enumerate() - self.assertIn(id, ids) - self.assertIn(main, ids) - self.assertEqual(len(ids), 3) + self.assertEqual(set(interpreters._enumerate()), {main, id1, id2}) def test_in_threaded_subinterpreter(self): main, = interpreters._enumerate() - id = interpreters.create() + id1 = interpreters.create() + ns = None + script = dedent(""" + import _interpreters + id = _interpreters.create() + """) def f(): - interpreters.run_string(id, dedent(""" - import _interpreters - _interpreters.create() - """)) + nonlocal ns + ns = interpreters.run_string_unrestricted(id1, script) t = threading.Thread(target=f) t.start() t.join() + id2 = ns['id'] - ids = interpreters._enumerate() - self.assertIn(id, ids) - self.assertIn(main, ids) - self.assertEqual(len(ids), 3) + self.assertEqual(set(interpreters._enumerate()), {main, id1, id2}) def test_after_destroy_all(self): @@ -214,21 +213,27 @@ def test_bad_id(self): interpreters.destroy(-1) def test_from_current(self): + main, = interpreters._enumerate() id = interpreters.create() + script = dedent(""" + import _interpreters + _interpreters.destroy({}) + """).format(id) + with self.assertRaises(RuntimeError): - interpreters.run_string(id, dedent(""" - import _interpreters - _interpreters.destroy({}) - """).format(id)) + interpreters.run_string(id, script) + self.assertEqual(set(interpreters._enumerate()), {main, id}) def test_from_sibling(self): main, = interpreters._enumerate() id1 = interpreters.create() id2 = interpreters.create() - interpreters.run_string(id1, dedent(""" + script = dedent(""" import _interpreters _interpreters.destroy({}) - """).format(id2)) + """).format(id2) + interpreters.run_string(id1, script) + self.assertEqual(set(interpreters._enumerate()), {main, id1}) def test_from_other_thread(self): @@ -241,6 +246,8 @@ def f(): t.join() def test_still_running(self): + # XXX Rewrite this test without files by using + # run_string_unrestricted(). main, = interpreters._enumerate() id = interpreters.create() def f(): @@ -428,5 +435,57 @@ def test_SystemError(self): self.assertEqual(cm.exception.code, 42) +class RunStringUnrestrictedTests(TestBase): + + def setUp(self): + self.id = interpreters.create() + + def test_without_ns(self): + script = dedent(""" + spam = 42 + """) + ns = interpreters.run_string_unrestricted(self.id, script) + + self.assertEqual(ns['spam'], 42) + + def test_with_ns(self): + updates = {'spam': 'ham', 'eggs': -1} + script = dedent(""" + spam = 42 + result = spam + eggs + """) + ns = interpreters.run_string_unrestricted(self.id, script, updates) + + self.assertEqual(ns['spam'], 42) + self.assertEqual(ns['eggs'], -1) + self.assertEqual(ns['result'], 41) + + def test_ns_does_not_overwrite(self): + updates = {'__name__': 'not __main__'} + script = dedent(""" + spam = 42 + """) + ns = interpreters.run_string_unrestricted(self.id, script, updates) + + self.assertEqual(ns['__name__'], '__main__') + + def test_return_execution_namespace(self): + script = dedent(""" + spam = 42 + """) + ns = interpreters.run_string_unrestricted(self.id, script) + + ns.pop('__builtins__') + ns.pop('__loader__') + self.assertEqual(ns, { + '__name__': '__main__', + '__annotations__': {}, + '__doc__': None, + '__package__': None, + '__spec__': None, + 'spam': 42, + }) + + if __name__ == '__main__': unittest.main() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index a46610e5a10899..47e4c3f1dcee40 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -89,14 +89,11 @@ _ensure_not_running(PyInterpreterState *interp) return 0; } -static int -_run_string(PyInterpreterState *interp, const char *codestr) +static PyObject * +_run_string(PyInterpreterState *interp, const char *codestr, PyObject *updates) { - PyObject *result = NULL; - PyObject *exc = NULL, *value = NULL, *tb = NULL; - if (_ensure_not_running(interp) < 0) - return -1; + return NULL; // Switch to interpreter. PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); @@ -104,19 +101,32 @@ _run_string(PyInterpreterState *interp, const char *codestr) // Run the string (see PyRun_SimpleStringFlags). // XXX How to handle sys.exit()? + PyObject *exc = NULL, *value = NULL, *tb = NULL; + PyObject *ns = NULL; + // XXX Force a fresh __main__ module? PyObject *m = PyImport_AddModule("__main__"); if (m == NULL) { PyErr_Fetch(&exc, &value, &tb); goto done; } - PyObject *d = PyModule_GetDict(m); - result = PyRun_StringFlags(codestr, Py_file_input, d, d, NULL); + ns = PyModule_GetDict(m); + if (ns == NULL) { + PyErr_Fetch(&exc, &value, &tb); + goto done; + } + if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) { + PyErr_Fetch(&exc, &value, &tb); + goto done; + } + PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); if (result == NULL) { + ns = NULL; // Get the exception from the subinterpreter. PyErr_Fetch(&exc, &value, &tb); goto done; } Py_DECREF(result); // We throw away the result. + Py_INCREF(ns); // It is a borrowed reference. done: // Switch back. @@ -126,7 +136,7 @@ _run_string(PyInterpreterState *interp, const char *codestr) // Propagate any exception out to the caller. PyErr_Restore(exc, value, tb); - return (result == NULL) ? -1 : 0; + return ns; } /* module level code ********************************************************/ @@ -273,32 +283,85 @@ interp_run_string(PyObject *self, PyObject *args) } // Run the code in the interpreter. - if (_run_string(interp, codestr) < 0) + PyObject *ns = _run_string(interp, codestr, NULL); + if (ns == NULL) return NULL; else Py_RETURN_NONE; } PyDoc_STRVAR(run_string_doc, -"run_string(ID, sourcetext) -> run_id\n\ +"run_string(ID, sourcetext)\n\ \n\ Execute the provided string in the identified interpreter.\n\ +\n\ +See PyRun_SimpleStrings."); + + +static PyObject * +interp_run_string_unrestricted(PyObject *self, PyObject *args) +{ + PyObject *id, *code, *ns = NULL; + if (!PyArg_UnpackTuple(args, "run_string_unrestricted", 2, 3, + &id, &code, &ns)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int"); + return NULL; + } + if (!PyUnicode_Check(code)) { + PyErr_SetString(PyExc_TypeError, + "second arg (code) must be a string"); + return NULL; + } + if (ns == Py_None) + ns = NULL; + + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + + // Extract code. + Py_ssize_t size; + const char *codestr = PyUnicode_AsUTF8AndSize(code, &size); + if (codestr == NULL) + return NULL; + if (strlen(codestr) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, + "source code string cannot contain null bytes"); + return NULL; + } + + // Run the code in the interpreter. + return _run_string(interp, codestr, ns); +} + +PyDoc_STRVAR(run_string_unrestricted_doc, +"run_string_unrestricted(ID, sourcetext, ns=None) -> main module ns\n\ +\n\ +Execute the provided string in the identified interpreter. Return the\n\ +dict in which the code executed. If the ns arg is provided then it is\n\ +merged into the execution namespace before the code is executed.\n\ +\n\ See PyRun_SimpleStrings."); static PyMethodDef module_functions[] = { - {"create", (PyCFunction)interp_create, + {"create", (PyCFunction)interp_create, METH_VARARGS, create_doc}, - {"destroy", (PyCFunction)interp_destroy, + {"destroy", (PyCFunction)interp_destroy, METH_VARARGS, destroy_doc}, - {"_enumerate", (PyCFunction)interp_enumerate, + {"_enumerate", (PyCFunction)interp_enumerate, METH_NOARGS, NULL}, - {"run_string", (PyCFunction)interp_run_string, + {"run_string", (PyCFunction)interp_run_string, METH_VARARGS, run_string_doc}, + {"run_string_unrestricted", (PyCFunction)interp_run_string_unrestricted, + METH_VARARGS, run_string_unrestricted_doc}, - {NULL, NULL} /* sentinel */ + {NULL, NULL} /* sentinel */ }; From 83685829ec903ec0e7d7bbd2fa01897dee117288 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jan 2017 18:17:18 -0700 Subject: [PATCH 08/20] Exit out of the child process. --- Lib/test/test__interpreters.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index edd279c16c4935..1402872e3412d6 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -357,6 +357,9 @@ def test_fork(self): with open(filename, 'w') as out: out.write('{}') os.write(w, b'done!') + + # Kill the unittest runner in the child process. + os._exit(1) else: import select try: @@ -365,7 +368,6 @@ def test_fork(self): os.close(r) os.close(w) """).format(filename, expected) - # XXX Kill the child process in a unittest-friendly way. interpreters.run_string(self.id, script) self.assert_file_contains(expected) From 9669ca75456b2dd07192075181ee29ea3d507017 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jan 2017 18:45:52 -0700 Subject: [PATCH 09/20] Resolve several TODOs. --- Doc/library/_interpreters.rst | 6 +++--- Modules/_interpretersmodule.c | 12 +++--------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index a4f86e38496e5b..e2d0191ac71703 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -50,9 +50,9 @@ It defines the following functions: threads do. If the code results in an exception then that exception is raised in the thread in which run_string() was called, similar to how :func:`exec` works. This aligns with how interpreters are not - inherently threaded. - -.. XXX sys.exit() (and SystemExit) is swallowed? + inherently threaded. Note that SystemExit (as raised by sys.exit()) + is not treated any differently and will result in the process ending + if not caught explicitly. .. function:: run_string_unrestricted(id, command, ns=None) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 47e4c3f1dcee40..a8d5b5391a1a64 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -45,7 +45,7 @@ _look_up(PyObject *requested_id) long long id = PyLong_AsLongLong(requested_id); if (id == -1 && PyErr_Occurred() != NULL) return NULL; - // XXX Fail if larger than INT64_MAX? + assert(id <= INT64_MAX); return _look_up_int64(id); } @@ -100,7 +100,6 @@ _run_string(PyInterpreterState *interp, const char *codestr, PyObject *updates) PyThreadState *save_tstate = PyThreadState_Swap(tstate); // Run the string (see PyRun_SimpleStringFlags). - // XXX How to handle sys.exit()? PyObject *exc = NULL, *value = NULL, *tb = NULL; PyObject *ns = NULL; // XXX Force a fresh __main__ module? @@ -141,8 +140,6 @@ _run_string(PyInterpreterState *interp, const char *codestr, PyObject *updates) /* module level code ********************************************************/ -// XXX track count? - static PyObject * interp_create(PyObject *self, PyObject *args) { @@ -205,10 +202,9 @@ interp_destroy(PyObject *self, PyObject *args) // Destroy the interpreter. //PyInterpreterState_Delete(interp); PyThreadState *tstate, *save_tstate; - tstate = PyInterpreterState_ThreadHead(interp); // XXX Is this the right one? + tstate = PyInterpreterState_ThreadHead(interp); save_tstate = PyThreadState_Swap(tstate); - // XXX Stop current execution? - Py_EndInterpreter(tstate); // XXX Handle possible errors? + Py_EndInterpreter(tstate); PyThreadState_Swap(save_tstate); Py_RETURN_NONE; @@ -229,8 +225,6 @@ interp_enumerate(PyObject *self) PyObject *ids, *id; PyInterpreterState *interp; - // XXX Handle multiple main interpreters. - ids = PyList_New(0); if (ids == NULL) return NULL; From 79edd9c7d05ec0e53dbdfed7e32333ccf2b95901 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jan 2017 19:17:53 -0700 Subject: [PATCH 10/20] Set up the execution namespace before switching threads. --- Modules/_interpretersmodule.c | 66 ++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index a8d5b5391a1a64..6366c7ec5e7611 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -89,45 +89,23 @@ _ensure_not_running(PyInterpreterState *interp) return 0; } -static PyObject * -_run_string(PyInterpreterState *interp, const char *codestr, PyObject *updates) +static int +_run_string(PyInterpreterState *interp, const char *codestr, PyObject *ns) { - if (_ensure_not_running(interp) < 0) - return NULL; - // Switch to interpreter. PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); PyThreadState *save_tstate = PyThreadState_Swap(tstate); // Run the string (see PyRun_SimpleStringFlags). PyObject *exc = NULL, *value = NULL, *tb = NULL; - PyObject *ns = NULL; - // XXX Force a fresh __main__ module? - PyObject *m = PyImport_AddModule("__main__"); - if (m == NULL) { - PyErr_Fetch(&exc, &value, &tb); - goto done; - } - ns = PyModule_GetDict(m); - if (ns == NULL) { - PyErr_Fetch(&exc, &value, &tb); - goto done; - } - if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) { - PyErr_Fetch(&exc, &value, &tb); - goto done; - } PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); if (result == NULL) { - ns = NULL; // Get the exception from the subinterpreter. PyErr_Fetch(&exc, &value, &tb); - goto done; + } else { + Py_DECREF(result); // We throw away the result. } - Py_DECREF(result); // We throw away the result. - Py_INCREF(ns); // It is a borrowed reference. -done: // Switch back. if (save_tstate != NULL) PyThreadState_Swap(save_tstate); @@ -135,6 +113,38 @@ _run_string(PyInterpreterState *interp, const char *codestr, PyObject *updates) // Propagate any exception out to the caller. PyErr_Restore(exc, value, tb); + return result == NULL ? -1 : 0; +} + +static PyObject * +_run_string_in_main(PyInterpreterState *interp, const char *codestr, + PyObject *updates) +{ + if (_ensure_not_running(interp) < 0) + return NULL; + + // Get the namespace in which to execute. + // XXX Force a fresh __main__ module? + PyObject *m = PyMapping_GetItemString(interp->modules, "__main__"); + if (m == NULL) + return NULL; + PyObject *orig = PyModule_GetDict(m); // borrowed + Py_DECREF(m); + if (orig == NULL) + return NULL; + PyObject *ns = PyDict_Copy(orig); + if (ns == NULL) + return NULL; + if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) { + Py_DECREF(ns); + return NULL; + } + + if (_run_string(interp, codestr, ns) < 0) { + Py_DECREF(ns); + return NULL; + } + return ns; } @@ -277,7 +287,7 @@ interp_run_string(PyObject *self, PyObject *args) } // Run the code in the interpreter. - PyObject *ns = _run_string(interp, codestr, NULL); + PyObject *ns = _run_string_in_main(interp, codestr, NULL); if (ns == NULL) return NULL; else @@ -328,7 +338,7 @@ interp_run_string_unrestricted(PyObject *self, PyObject *args) } // Run the code in the interpreter. - return _run_string(interp, codestr, ns); + return _run_string_in_main(interp, codestr, ns); } PyDoc_STRVAR(run_string_unrestricted_doc, From 17b0e282f8ae6dbd3cbf59df74ba5a5b3509a303 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 3 Jan 2017 23:42:29 -0700 Subject: [PATCH 11/20] Run in a copy of __main__. --- Lib/test/test__interpreters.py | 11 +++++- Modules/_interpretersmodule.c | 70 +++++++++++++++++++++++++++------- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 1402872e3412d6..3012f4e59ea83d 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -300,7 +300,7 @@ def _empty_file(self): def assert_file_contains(self, expected, filename=None): if filename is None: filename = self.filename - self.assertIsNot(filename, None) + self.assertIsNotNone(filename) with open(filename) as out: content = out.read() self.assertEqual(content, expected) @@ -471,6 +471,15 @@ def test_ns_does_not_overwrite(self): self.assertEqual(ns['__name__'], '__main__') + def test_main_not_shared(self): + ns1 = interpreters.run_string_unrestricted(self.id, 'spam = True') + ns2 = interpreters.run_string_unrestricted(self.id, 'eggs = False') + + self.assertIn('spam', ns1) + self.assertNotIn('eggs', ns1) + self.assertIn('eggs', ns2) + self.assertNotIn('spam', ns2) + def test_return_execution_namespace(self): script = dedent(""" spam = 42 diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 6366c7ec5e7611..a5146cda48af15 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -89,6 +89,33 @@ _ensure_not_running(PyInterpreterState *interp) return 0; } +static PyObject * +_copy_module(PyObject *m, PyObject *updates) +{ + PyObject *orig = PyModule_GetDict(m); // borrowed + if (orig == NULL) { + return NULL; + } + + PyObject *copy = PyModule_New("__main__"); + if (copy == NULL) { + return NULL; + } + PyObject *ns = PyModule_GetDict(copy); // borrowed + if (ns == NULL) + goto error; + + if (PyDict_Merge(ns, orig, 1) < 0) + goto error; + if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) + goto error; + return copy; + +error: + Py_DECREF(copy); + return NULL; +} + static int _run_string(PyInterpreterState *interp, const char *codestr, PyObject *ns) { @@ -123,31 +150,46 @@ _run_string_in_main(PyInterpreterState *interp, const char *codestr, if (_ensure_not_running(interp) < 0) return NULL; - // Get the namespace in which to execute. - // XXX Force a fresh __main__ module? - PyObject *m = PyMapping_GetItemString(interp->modules, "__main__"); - if (m == NULL) + // Get the namespace in which to execute. This involves creating + // a new module, updating it from the __main__ module and the given + // updates (if any), replacing the __main__ with the new module in + // sys.modules, and then using the new module's __dict__. At the + // end we restore the original __main__ module. + PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); + if (main_mod == NULL) return NULL; - PyObject *orig = PyModule_GetDict(m); // borrowed - Py_DECREF(m); - if (orig == NULL) - return NULL; - PyObject *ns = PyDict_Copy(orig); - if (ns == NULL) + PyObject *m = _copy_module(main_mod, updates); + if (m == NULL) { + Py_DECREF(main_mod); return NULL; - if (updates != NULL && PyDict_Merge(ns, updates, 0) < 0) { - Py_DECREF(ns); + } + if (PyMapping_SetItemString(interp->modules, "__main__", m) < 0) { + Py_DECREF(main_mod); + Py_DECREF(m); return NULL; } + PyObject *ns = PyModule_GetDict(m); // borrowed + Py_INCREF(ns); + Py_DECREF(m); + // Run the string. + PyObject *result = ns; if (_run_string(interp, codestr, ns) < 0) { + result = NULL; Py_DECREF(ns); - return NULL; } - return ns; + // Restore __main__. + if (PyMapping_SetItemString(interp->modules, "__main__", main_mod) < 0) { + // XXX Chain exceptions... + //PyErr_Restore(exc, value, tb); + } + Py_DECREF(main_mod); + + return result; } + /* module level code ********************************************************/ static PyObject * From 7d8f744f2202a8e27a7924c29bac250baf0fcf6c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 11:58:31 -0700 Subject: [PATCH 12/20] Close stdin and stdout after the proc finishes. --- Lib/test/test__interpreters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 3012f4e59ea83d..93c1ee58a7e19f 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -54,8 +54,8 @@ def test_still_running_at_exit(self): time.sleep(1_000_000) """) filename = script_helper.make_script(self.dirname, 'interp', script) - proc = script_helper.spawn_python(filename) - retcode = proc.wait() + with script_helper.spawn_python(filename) as proc: + retcode = proc.wait() self.assertEqual(retcode, 0) From 80ab76a6086982339f09e008fcaa1c8993f701d0 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 12:12:07 -0700 Subject: [PATCH 13/20] Clean up a test. --- Lib/test/test__interpreters.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 93c1ee58a7e19f..459ff8650a8470 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -3,7 +3,7 @@ import os.path import shutil import tempfile -from textwrap import dedent +from textwrap import dedent, indent import threading import unittest @@ -14,10 +14,13 @@ SCRIPT_THREADED_INTERP = """\ +from textwrap import dedent import threading import _interpreters def f(): - _interpreters.run_string(id, {!r}) + _interpreters.run_string(id, dedent(''' + {} + ''')) t = threading.Thread(target=f) t.start() @@ -48,11 +51,12 @@ def tearDown(self): shutil.rmtree(self.dirname) def test_still_running_at_exit(self): - script = SCRIPT_THREADED_INTERP.format("""if True: + subscript = dedent(""" import time # Give plenty of time for the main interpreter to finish. time.sleep(1_000_000) """) + script = SCRIPT_THREADED_INTERP.format(indent(subscript, ' ')) filename = script_helper.make_script(self.dirname, 'interp', script) with script_helper.spawn_python(filename) as proc: retcode = proc.wait() From 7afda96be6a662275af43a130472726c5624bbb4 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 13:10:53 -0700 Subject: [PATCH 14/20] Chain exceptions during cleanup. --- Modules/_interpretersmodule.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index a5146cda48af15..ff08ae28c449dd 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -180,9 +180,12 @@ _run_string_in_main(PyInterpreterState *interp, const char *codestr, } // Restore __main__. + PyObject *exc = NULL, *value = NULL, *tb = NULL; + PyErr_Fetch(&exc, &value, &tb); if (PyMapping_SetItemString(interp->modules, "__main__", main_mod) < 0) { - // XXX Chain exceptions... - //PyErr_Restore(exc, value, tb); + _PyErr_ChainExceptions(exc, value, tb); + } else { + PyErr_Restore(exc, value, tb); } Py_DECREF(main_mod); From 4e07f7ab07b3667f429aa6b3fb18941030a489ed Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 13:31:50 -0700 Subject: [PATCH 15/20] Finish the module docs. --- Doc/library/_interpreters.rst | 36 +++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index e2d0191ac71703..6e79277895936e 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -6,21 +6,30 @@ .. versionadded:: 3,7 - :ref:`_sub-interpreter-support` - -threading - -------------- This module provides low-level primitives for working with multiple -Python interpreters in the same process. - +Python interpreters in the same runtime in the current process. .. XXX The :mod:`interpreters` module provides an easier to use and higher-level API built on top of this module. +More information about (sub)interpreters is found at +:ref:`_sub-interpreter-support`, including what data is shared between +interpreters and what is unique. Note particularly that interpreters +aren't inherently threaded, even though they track and manage Python +threads. To run code in an interpreter in a different OS thread, call +:func:`run_string` in a function that you run in a new Python thread. +For example:: + + id = _interpreters.create() + def f(): + _interpreters.run_string(id, 'print("in a thread")') + + t = threading.Thread(target=f) + t.start() + This module is optional. It is provided by Python implementations which support multiple interpreters. - .. XXX For systems lacking the :mod:`_interpreters` module, the :mod:`_dummy_interpreters` module is available. It duplicates this module's interface and can be used as a drop-in replacement. @@ -42,9 +51,10 @@ It defines the following functions: .. function:: run_string(id, command) A wrapper around :c:func:`PyRun_SimpleString` which runs the provided - Python program using the identified interpreter. Providing an - invalid or unknown ID results in a RuntimeError, likewise if the main - interpreter or any other running interpreter is used. + Python program in the main thread of the identified interpreter. + Providing an invalid or unknown ID results in a RuntimeError, + likewise if the main interpreter or any other running interpreter + is used. Any value returned from the code is thrown away, similar to what threads do. If the code results in an exception then that exception @@ -62,9 +72,3 @@ It defines the following functions: merged into the execution namespace before execution. Note that this allows objects to leak between interpreters, which may not be desirable. - - -**Caveats:** - -* ... - From 060a720a7c36a0a3119fa586a6e1d54a96c50f2d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 23 May 2017 16:41:55 -0700 Subject: [PATCH 16/20] Fix docs. --- Doc/library/_interpreters.rst | 7 +------ Doc/library/concurrency.rst | 1 + 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 6e79277895936e..445926ac4b87ba 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -10,11 +10,9 @@ This module provides low-level primitives for working with multiple Python interpreters in the same runtime in the current process. -.. XXX The :mod:`interpreters` module provides an easier to use and - higher-level API built on top of this module. More information about (sub)interpreters is found at -:ref:`_sub-interpreter-support`, including what data is shared between +:ref:`sub-interpreter-support`, including what data is shared between interpreters and what is unique. Note particularly that interpreters aren't inherently threaded, even though they track and manage Python threads. To run code in an interpreter in a different OS thread, call @@ -30,9 +28,6 @@ For example:: This module is optional. It is provided by Python implementations which support multiple interpreters. -.. XXX For systems lacking the :mod:`_interpreters` module, the - :mod:`_dummy_interpreters` module is available. It duplicates this - module's interface and can be used as a drop-in replacement. It defines the following functions: diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index 0de281bd149531..76b9c52f593555 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -29,3 +29,4 @@ The following are support modules for some of the above services: dummy_threading.rst _thread.rst _dummy_thread.rst + _interpreters.rst From a85df93ace838fbf2c3e4d5f365b5adbcf94e89e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 Dec 2016 15:30:43 -0700 Subject: [PATCH 17/20] Add get_current() and enumerate(). --- Doc/library/_interpreters.rst | 10 ++++ Lib/test/test__interpreters.py | 88 ++++++++++++++++++++++++---------- Modules/_interpretersmodule.c | 27 ++++++++++- 3 files changed, 97 insertions(+), 28 deletions(-) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 445926ac4b87ba..8c58d22cca79df 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -31,6 +31,16 @@ support multiple interpreters. It defines the following functions: +.. function:: enumerate() + + Return a list of the IDs of every existing interpreter. + + +.. function:: get_current() + + Return the ID of the currently running interpreter. + + .. function:: create() Initialize a new Python interpreter and return its identifier. The diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 459ff8650a8470..c0836299b83117 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -67,7 +67,7 @@ def test_still_running_at_exit(self): class TestBase(unittest.TestCase): def tearDown(self): - for id in interpreters._enumerate(): + for id in interpreters.enumerate(): if id == 0: # main continue try: @@ -76,13 +76,49 @@ def tearDown(self): pass # already destroyed +class EnumerateTests(TestBase): + + def test_multiple(self): + main, = interpreters.enumerate() + id1 = interpreters.create() + id2 = interpreters.create() + ids = interpreters.enumerate() + + self.assertEqual(set(ids), {main, id1, id2}) + + def test_main_only(self): + main, = interpreters.enumerate() + + self.assertEqual(main, 0) + + +class GetCurrentTests(TestBase): + + def test_main(self): + main, = interpreters.enumerate() + id = interpreters.get_current() + + self.assertEqual(id, main) + + def test_sub(self): + id1 = interpreters.create() + ns = interpreters.run_string_unrestricted(id1, dedent(""" + import _interpreters + id = _interpreters.get_current() + """)) + id2 = ns['id'] + + self.assertEqual(id2, id1) + + class CreateTests(TestBase): def test_in_main(self): id = interpreters.create() - self.assertIn(id, interpreters._enumerate()) + self.assertIn(id, interpreters.enumerate()) + @unittest.skip('enable this test when working on pystate.c') def test_unique_id(self): seen = set() for _ in range(100): @@ -105,10 +141,10 @@ def f(): with lock: t.start() t.join() - self.assertIn(id, interpreters._enumerate()) + self.assertIn(id, interpreters.enumerate()) def test_in_subinterpreter(self): - main, = interpreters._enumerate() + main, = interpreters.enumerate() id1 = interpreters.create() ns = interpreters.run_string_unrestricted(id1, dedent(""" import _interpreters @@ -116,10 +152,10 @@ def test_in_subinterpreter(self): """)) id2 = ns['id'] - self.assertEqual(set(interpreters._enumerate()), {main, id1, id2}) + self.assertEqual(set(interpreters.enumerate()), {main, id1, id2}) def test_in_threaded_subinterpreter(self): - main, = interpreters._enumerate() + main, = interpreters.enumerate() id1 = interpreters.create() ns = None script = dedent(""" @@ -135,11 +171,11 @@ def f(): t.join() id2 = ns['id'] - self.assertEqual(set(interpreters._enumerate()), {main, id1, id2}) + self.assertEqual(set(interpreters.enumerate()), {main, id1, id2}) def test_after_destroy_all(self): - before = set(interpreters._enumerate()) + before = set(interpreters.enumerate()) # Create 3 subinterpreters. ids = [] for _ in range(3): @@ -150,10 +186,10 @@ def test_after_destroy_all(self): interpreters.destroy(id) # Finally, create another. id = interpreters.create() - self.assertEqual(set(interpreters._enumerate()), before | {id}) + self.assertEqual(set(interpreters.enumerate()), before | {id}) def test_after_destroy_some(self): - before = set(interpreters._enumerate()) + before = set(interpreters.enumerate()) # Create 3 subinterpreters. id1 = interpreters.create() id2 = interpreters.create() @@ -163,7 +199,7 @@ def test_after_destroy_some(self): interpreters.destroy(id3) # Finally, create another. id = interpreters.create() - self.assertEqual(set(interpreters._enumerate()), before | {id, id2}) + self.assertEqual(set(interpreters.enumerate()), before | {id, id2}) class DestroyTests(TestBase): @@ -172,25 +208,25 @@ def test_one(self): id1 = interpreters.create() id2 = interpreters.create() id3 = interpreters.create() - self.assertIn(id2, interpreters._enumerate()) + self.assertIn(id2, interpreters.enumerate()) interpreters.destroy(id2) - self.assertNotIn(id2, interpreters._enumerate()) - self.assertIn(id1, interpreters._enumerate()) - self.assertIn(id3, interpreters._enumerate()) + self.assertNotIn(id2, interpreters.enumerate()) + self.assertIn(id1, interpreters.enumerate()) + self.assertIn(id3, interpreters.enumerate()) def test_all(self): - before = set(interpreters._enumerate()) + before = set(interpreters.enumerate()) ids = set() for _ in range(3): id = interpreters.create() ids.add(id) - self.assertEqual(set(interpreters._enumerate()), before | ids) + self.assertEqual(set(interpreters.enumerate()), before | ids) for id in ids: interpreters.destroy(id) - self.assertEqual(set(interpreters._enumerate()), before) + self.assertEqual(set(interpreters.enumerate()), before) def test_main(self): - main, = interpreters._enumerate() + main, = interpreters.enumerate() with self.assertRaises(RuntimeError): interpreters.destroy(main) @@ -217,7 +253,7 @@ def test_bad_id(self): interpreters.destroy(-1) def test_from_current(self): - main, = interpreters._enumerate() + main, = interpreters.enumerate() id = interpreters.create() script = dedent(""" import _interpreters @@ -226,10 +262,10 @@ def test_from_current(self): with self.assertRaises(RuntimeError): interpreters.run_string(id, script) - self.assertEqual(set(interpreters._enumerate()), {main, id}) + self.assertEqual(set(interpreters.enumerate()), {main, id}) def test_from_sibling(self): - main, = interpreters._enumerate() + main, = interpreters.enumerate() id1 = interpreters.create() id2 = interpreters.create() script = dedent(""" @@ -238,7 +274,7 @@ def test_from_sibling(self): """).format(id2) interpreters.run_string(id1, script) - self.assertEqual(set(interpreters._enumerate()), {main, id1}) + self.assertEqual(set(interpreters.enumerate()), {main, id1}) def test_from_other_thread(self): id = interpreters.create() @@ -252,7 +288,7 @@ def f(): def test_still_running(self): # XXX Rewrite this test without files by using # run_string_unrestricted(). - main, = interpreters._enumerate() + main, = interpreters.enumerate() id = interpreters.create() def f(): interpreters.run_string(id, wait_script) @@ -265,7 +301,7 @@ def f(): interpreters.destroy(id) t.join() - self.assertEqual(set(interpreters._enumerate()), {main, id}) + self.assertEqual(set(interpreters.enumerate()), {main, id}) class RunStringTests(TestBase): @@ -389,7 +425,7 @@ def f(): def test_does_not_exist(self): id = 0 - while id in interpreters._enumerate(): + while id in interpreters.enumerate(): id += 1 with self.assertRaises(RuntimeError): interpreters.run_string(id, 'print("spam")') diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index ff08ae28c449dd..1a1dbf6b0d3c6d 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -299,6 +299,27 @@ interp_enumerate(PyObject *self) return ids; } +PyDoc_STRVAR(enumerate_doc, +"enumerate() -> [ID]\n\ +\n\ +Return a list containing the ID of every existing interpreter."); + + +static PyObject * +interp_get_current(PyObject *self) +{ + PyInterpreterState *interp =_get_current(); + if (interp == NULL) + return NULL; + return _get_id(interp); +} + +PyDoc_STRVAR(get_current_doc, +"get_current() -> ID\n\ +\n\ +Return the ID of current interpreter."); + + static PyObject * interp_run_string(PyObject *self, PyObject *args) { @@ -402,8 +423,10 @@ static PyMethodDef module_functions[] = { {"destroy", (PyCFunction)interp_destroy, METH_VARARGS, destroy_doc}, - {"_enumerate", (PyCFunction)interp_enumerate, - METH_NOARGS, NULL}, + {"enumerate", (PyCFunction)interp_enumerate, + METH_NOARGS, enumerate_doc}, + {"get_current", (PyCFunction)interp_get_current, + METH_NOARGS, get_current_doc}, {"run_string", (PyCFunction)interp_run_string, METH_VARARGS, run_string_doc}, From f605ae7cae69b0c188274b603ee33405c4b7c0f9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 29 Dec 2016 15:33:17 -0700 Subject: [PATCH 18/20] Add is_running(). --- Doc/library/_interpreters.rst | 6 ++++++ Lib/test/test__interpreters.py | 30 ++++++++++++++++++++++++++++++ Modules/_interpretersmodule.c | 30 ++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 8c58d22cca79df..1bb107e500e4a5 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -41,6 +41,12 @@ It defines the following functions: Return the ID of the currently running interpreter. +.. function:: is_running(id) + + Return whether or not the identified interpreter is currently + running any code. + + .. function:: create() Initialize a new Python interpreter and return its identifier. The diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index c0836299b83117..49ed4444be84ee 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -111,6 +111,36 @@ def test_sub(self): self.assertEqual(id2, id1) +class IsRunningTests(TestBase): + + def test_main_running(self): + main, = interpreters.enumerate() + sub = interpreters.create() + main_running = interpreters.is_running(main) + sub_running = interpreters.is_running(sub) + + self.assertTrue(main_running) + self.assertFalse(sub_running) + + def test_sub_running(self): + main, = interpreters.enumerate() + sub1 = interpreters.create() + sub2 = interpreters.create() + ns = interpreters.run_string_unrestricted(sub1, dedent(f""" + import _interpreters + main = _interpreters.is_running({main}) + sub1 = _interpreters.is_running({sub1}) + sub2 = _interpreters.is_running({sub2}) + """)) + main_running = ns['main'] + sub1_running = ns['sub1'] + sub2_running = ns['sub2'] + + self.assertTrue(main_running) + self.assertTrue(sub1_running) + self.assertFalse(sub2_running) + + class CreateTests(TestBase): def test_in_main(self): diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 1a1dbf6b0d3c6d..83c5ace03f95b8 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -417,6 +417,34 @@ merged into the execution namespace before the code is executed.\n\ See PyRun_SimpleStrings."); +static PyObject * +interp_is_running(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "is_running", 1, 1, &id)) + return NULL; + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "ID must be an int"); + return NULL; + } + + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) + return NULL; + int is_running = _is_running(interp); + if (is_running < 0) + return NULL; + if (is_running) + Py_RETURN_TRUE; + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(is_running_doc, +"is_running(id) -> bool\n\ +\n\ +Return whether or not the identified interpreter is running."); + + static PyMethodDef module_functions[] = { {"create", (PyCFunction)interp_create, METH_VARARGS, create_doc}, @@ -427,6 +455,8 @@ static PyMethodDef module_functions[] = { METH_NOARGS, enumerate_doc}, {"get_current", (PyCFunction)interp_get_current, METH_NOARGS, get_current_doc}, + {"is_running", (PyCFunction)interp_is_running, + METH_VARARGS, is_running_doc}, {"run_string", (PyCFunction)interp_run_string, METH_VARARGS, run_string_doc}, From 9704487b3d726bb5d91d87ed0859a5ff6ed9cfce Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 14:41:58 -0700 Subject: [PATCH 19/20] Add get_main(). --- Doc/library/_interpreters.rst | 5 +++++ Lib/test/test__interpreters.py | 10 ++++++++++ Modules/_interpretersmodule.c | 15 +++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/Doc/library/_interpreters.rst b/Doc/library/_interpreters.rst index 1bb107e500e4a5..bbc945016e2880 100644 --- a/Doc/library/_interpreters.rst +++ b/Doc/library/_interpreters.rst @@ -41,6 +41,11 @@ It defines the following functions: Return the ID of the currently running interpreter. +.. function:: get_main() + + Return the ID of the main interpreter. + + .. function:: is_running(id) Return whether or not the identified interpreter is currently diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 49ed4444be84ee..29d04aa4d00cfb 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -111,6 +111,16 @@ def test_sub(self): self.assertEqual(id2, id1) +class GetMainTests(TestBase): + + def test_main(self): + expected, = interpreters.enumerate() + main = interpreters.get_main() + + self.assertEqual(main, 0) + self.assertEqual(main, expected) + + class IsRunningTests(TestBase): def test_main_running(self): diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 83c5ace03f95b8..5eef04d5d755a5 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -320,6 +320,19 @@ PyDoc_STRVAR(get_current_doc, Return the ID of current interpreter."); +static PyObject * +interp_get_main(PyObject *self) +{ + // Currently, 0 is always the main interpreter. + return PyLong_FromLongLong(0); +} + +PyDoc_STRVAR(get_main_doc, +"get_main() -> ID\n\ +\n\ +Return the ID of main interpreter."); + + static PyObject * interp_run_string(PyObject *self, PyObject *args) { @@ -455,6 +468,8 @@ static PyMethodDef module_functions[] = { METH_NOARGS, enumerate_doc}, {"get_current", (PyCFunction)interp_get_current, METH_NOARGS, get_current_doc}, + {"get_main", (PyCFunction)interp_get_main, + METH_NOARGS, get_main_doc}, {"is_running", (PyCFunction)interp_is_running, METH_VARARGS, is_running_doc}, From 7dadbe5d46d2f494c145082caf34c8bd80896c1c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 4 Jan 2017 18:16:31 -0700 Subject: [PATCH 20/20] Add the interpreters module. --- Doc/library/interpreters.rst | 64 +++++++++++++ Lib/interpreters.py | 79 ++++++++++++++++ Lib/test/test_interpreters.py | 167 ++++++++++++++++++++++++++++++++++ 3 files changed, 310 insertions(+) create mode 100644 Doc/library/interpreters.rst create mode 100644 Lib/interpreters.py create mode 100644 Lib/test/test_interpreters.py diff --git a/Doc/library/interpreters.rst b/Doc/library/interpreters.rst new file mode 100644 index 00000000000000..bef62d30a673fe --- /dev/null +++ b/Doc/library/interpreters.rst @@ -0,0 +1,64 @@ +:mod:`interpreters` --- high-level interpreters API +=================================================== + +.. module:: interpreters + :synopsis: High-level interpreters API. + +**Source code:** :source:`Lib/interpreters.py` + +.. versionadded:: 3,7 + +-------------- + +This module provides high-level interfaces for interacting with +the Python interpreters. It is built on top of the lower- level +:mod:`_interpreters` module. + +.. XXX Summarize :ref:`_sub-interpreter-support` here. + +This module defines the following functions: + +.. function:: enumerate() + + Return a list of all existing interpreters. + + +.. function:: get_current() + + Return the currently running interpreter. + + +.. function:: get_main() + + Return the main interpreter. + + +.. function:: create() + + Initialize a new Python interpreter and return it. The + interpreter will be created in the current thread and will remain + idle until something is run in it. + + +This module also defines the following class: + +.. class:: Interpreter(id) + + ``id`` is the interpreter's ID. + + .. property:: id + + The interpreter's ID. + + .. method:: is_running() + + Return whether or not the interpreter is currently running. + + .. method:: destroy() + + Finalize and destroy the interpreter. + + .. method:: run(code) + + Run the provided Python code in the interpreter, in the current + OS thread. Supported code: source text. diff --git a/Lib/interpreters.py b/Lib/interpreters.py new file mode 100644 index 00000000000000..3bfc567b6a79b0 --- /dev/null +++ b/Lib/interpreters.py @@ -0,0 +1,79 @@ +"""Interpreters module providing a high-level API to Python interpreters.""" + +import _interpreters + + +__all__ = ['enumerate', 'current', 'main', 'create', 'Interpreter'] + + +def enumerate(): + """Return a list of all existing interpreters.""" + return [Interpreter(id) + for id in _interpreters.enumerate()] + + +def current(): + """Return the currently running interpreter.""" + id = _interpreters.get_current() + return Interpreter(id) + + +def main(): + """Return the main interpreter.""" + id = _interpreters.get_main() + return Interpreter(id) + + +def create(): + """Return a new Interpreter. + + The interpreter is created in the current OS thread. It will remain + idle until its run() method is called. + """ + id = _interpreters.create() + return Interpreter(id) + + +class Interpreter: + """A single Python interpreter in the current runtime.""" + + def __init__(self, id): + self._id = id + + def __repr__(self): + return '{}(id={!r})'.format(type(self).__name__, self._id) + + def __eq__(self, other): + try: + other_id = other.id + except AttributeError: + return False + return self._id == other_id + + @property + def id(self): + """The interpreter's ID.""" + return self._id + + def is_running(self): + """Return whether or not the interpreter is currently running.""" + return _interpreters.is_running(self._id) + + def destroy(self): + """Finalize and destroy the interpreter. + + After calling destroy(), all operations on this interpreter + will fail. + """ + _interpreters.destroy(self._id) + + def run(self, code): + """Run the provided Python code in this interpreter. + + If the code is a string then it will be run as it would by + calling exec(). Other kinds of code are not supported. + """ + if isinstance(code, str): + _interpreters.run_string(self._id, code) + else: + raise NotImplementedError diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py new file mode 100644 index 00000000000000..3e020139e0bdae --- /dev/null +++ b/Lib/test/test_interpreters.py @@ -0,0 +1,167 @@ +import ast +import os.path +import shutil +import tempfile +from textwrap import dedent +import threading +import unittest + +from test import support + +_interpreters = support.import_module('_interpreters') +interpreters = support.import_module('interpreters') + + +class TestBase(unittest.TestCase): + + def setUp(self): + self.dirname = None + + def tearDown(self): + if self.dirname is not None: + try: + shutil.rmtree(self.dirname) + except FileNotFoundError: + pass # already deleted + + for id in _interpreters.enumerate(): + if id == 0: # main + continue + try: + _interpreters.destroy(id) + except RuntimeError: + pass # already destroyed + + def _resolve_filename(self, name): + if self.dirname is None: + self.dirname = tempfile.mkdtemp() + return os.path.join(self.dirname, name) + + def _empty_file(self, name): + filename = self._resolve_filename(name) + support.create_empty_file(filename) + return filename + + def assert_file_contains(self, filename, expected): + with open(filename) as out: + content = out.read() + self.assertEqual(content, expected) + + +class ModuleTests(TestBase): + + def test_enumerate(self): + main_id = _interpreters.get_main() + got = interpreters.enumerate() + self.assertEqual(set(i.id for i in got), {main_id}) + + id1 = _interpreters.create() + id2 = _interpreters.create() + got = interpreters.enumerate() + self.assertEqual(set(i.id for i in got), {main_id, id1, id2}) + + def test_current(self): + main_id = _interpreters.get_main() + current = interpreters.current() + self.assertEqual(current.id, main_id) + + id = _interpreters.create() + script = dedent(""" + import interpreters + interp = interpreters.current() + """) + ns = _interpreters.run_string_unrestricted(id, script) + current = ns['interp'] + self.assertEqual(current.id, id) + + def test_main(self): + expected = _interpreters.get_main() + main = interpreters.main() + + self.assertEqual(main.id, expected) + + def test_create(self): + main_id = _interpreters.get_main() + interp = interpreters.create() + + self.assertIsInstance(interp, interpreters.Interpreter) + self.assertGreater(interp.id, main_id) + + +class InterpreterTests(TestBase): + + def test_repr(self): + interp = interpreters.Interpreter(10) + result = repr(interp) + + self.assertEqual(result, 'Interpreter(id=10)') + + def test_equality(self): + interp1 = interpreters.Interpreter(0) + interp2 = interpreters.Interpreter(10) + interp3 = interpreters.Interpreter(10) + different = (interp1 == interp2) + same = (interp2 == interp3) + identity = (interp1 == interp1) + + self.assertFalse(different) + self.assertTrue(same) + self.assertTrue(identity) + + def test_id(self): + interp = interpreters.Interpreter(10) + id = interp.id + + self.assertEqual(id, 10) + + def test_is_running(self): + interp = interpreters.create() + is_running = interp.is_running() + self.assertFalse(is_running) + + script = 'is_running = interp.is_running()' + ns = _interpreters.run_string_unrestricted(interp.id, script, { + 'interp': interp, + }) + is_running = ns['is_running'] + self.assertTrue(is_running) + + def test_destroy(self): + before = set(_interpreters.enumerate()) + interp = interpreters.create() + self.assertEqual(set(_interpreters.enumerate()), before | {interp.id}) + + interp.destroy() + after = set(_interpreters.enumerate()) + + self.assertEqual(before, after) + + def test_run_string(self): + filename = self._empty_file('spam') + data = 'success!' + script = dedent(f""" + with open('{filename}', 'w') as out: + out.write('{data}') + """) + interp = interpreters.create() + + interp.run(script) + self.assert_file_contains(filename, data) + + def test_run_unsupported(self): + script = 'raise Exception' + interp = interpreters.create() + + with self.assertRaises(NotImplementedError): + interp.run(None) + with self.assertRaises(NotImplementedError): + interp.run(10) + node = compile(script, '