diff --git a/CMakeLists.txt b/CMakeLists.txt index caaf53499f3..008a3058334 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -223,6 +223,9 @@ include (MakeDistFiles) # Enable CTest include (CTest) +if (BUILD_TESTING) + include (TestFixtures) +endif () # Ensure the default behavior: don't ignore RPATH settings. set (CMAKE_SKIP_BUILD_RPATH OFF) diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index faf4fe295b5..4c3467dcf27 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -15,6 +15,7 @@ set (build_cmake_MODULES Sanitizers.cmake CCache.cmake LLDLinker.cmake + TestFixtures.cmake ) set_local_dist (build_cmake_DIST_local diff --git a/build/cmake/LoadTests.cmake b/build/cmake/LoadTests.cmake index d310899fedb..e6724165cd5 100644 --- a/build/cmake/LoadTests.cmake +++ b/build/cmake/LoadTests.cmake @@ -26,6 +26,12 @@ endif () # Split lines on newlines string (REPLACE "\n" ";" lines "${tests_out}") +# TODO: Allow individual test cases to specify the fixtures they want. +set (all_fixtures "mongoc/fixtures/fake_imds") +set (all_env + MCD_TEST_AZURE_IMDS_HOST=localhost:14987 # Refer: Fixtures.cmake + ) + # Generate the test definitions foreach (line IN LISTS lines) if (NOT line MATCHES "^/") @@ -44,5 +50,7 @@ foreach (line IN LISTS lines) SKIP_REGULAR_EXPRESSION "@@ctest-skipped@@" # 45 seconds of timeout on each test. TIMEOUT 45 + FIXTURES_REQUIRED "${all_fixtures}" + ENVIRONMENT "${all_env}" ) endforeach () diff --git a/build/cmake/TestFixtures.cmake b/build/cmake/TestFixtures.cmake new file mode 100644 index 00000000000..551aafb0a09 --- /dev/null +++ b/build/cmake/TestFixtures.cmake @@ -0,0 +1,49 @@ +find_package (Python3 COMPONENTS Interpreter) + +if (NOT TARGET Python3::Interpreter) + message (STATUS "Python3 was not found, so test fixtures will not be defined") + return () +endif () + +get_filename_component(_MONGOC_BUILD_SCRIPT_DIR "${CMAKE_CURRENT_LIST_DIR}" DIRECTORY) +set (_MONGOC_PROC_CTL_COMMAND "$" -u -- "${_MONGOC_BUILD_SCRIPT_DIR}/proc-ctl.py") + + +function (mongo_define_subprocess_fixture name) + cmake_parse_arguments(PARSE_ARGV 1 ARG "" "SPAWN_WAIT;STOP_WAIT;WORKING_DIRECTORY" "COMMAND") + string (MAKE_C_IDENTIFIER ident "${name}") + if (NOT ARG_SPAWN_WAIT) + set (ARG_SPAWN_WAIT 1) + endif () + if (NOT ARG_STOP_WAIT) + set (ARG_STOP_WAIT 5) + endif () + if (NOT ARG_WORKING_DIRECTORY) + set (ARG_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + endif () + if (NOT ARG_COMMAND) + message (SEND_ERROR "mongo_define_subprocess_fixture(${name}) requires a COMMAND") + return () + endif () + get_filename_component (ctl_dir "${CMAKE_CURRENT_BINARY_DIR}/${ident}.ctl" ABSOLUTE) + add_test (NAME "${name}/start" + COMMAND ${_MONGOC_PROC_CTL_COMMAND} start + "--ctl-dir=${ctl_dir}" + "--cwd=${ARG_WORKING_DIRECTORY}" + "--spawn-wait=${ARG_SPAWN_WAIT}" + -- ${ARG_COMMAND}) + add_test (NAME "${name}/stop" + COMMAND ${_MONGOC_PROC_CTL_COMMAND} stop "--ctl-dir=${ctl_dir}" --if-not-running=ignore) + set_property (TEST "${name}/start" PROPERTY FIXTURES_SETUP "${name}") + set_property (TEST "${name}/stop" PROPERTY FIXTURES_CLEANUP "${name}") +endfunction () + +# Create a fixture that runs a fake Azure IMDS server +mongo_define_subprocess_fixture( + mongoc/fixtures/fake_imds + SPAWN_WAIT 0.2 + COMMAND + "$" -u -- + "${_MONGOC_BUILD_SCRIPT_DIR}/bottle.py" fake_azure:imds + --bind localhost:14987 # Port 14987 chosen arbitrarily + ) diff --git a/build/fake_azure.py b/build/fake_azure.py index 16b408e5057..5ca6d7998ba 100644 --- a/build/fake_azure.py +++ b/build/fake_azure.py @@ -1,19 +1,21 @@ -from __future__ import annotations - +import functools +import json import sys import time -import json import traceback -import functools +from pathlib import Path + import bottle -from bottle import Bottle, HTTPResponse, request +from bottle import Bottle, HTTPResponse imds = Bottle(autojson=True) """An Azure IMDS server""" -from typing import TYPE_CHECKING, Any, Callable, Iterable, overload +from typing import TYPE_CHECKING, Any, Callable, Iterable, cast, overload -if TYPE_CHECKING: +if not TYPE_CHECKING: + from bottle import request +else: from typing import Protocol class _RequestParams(Protocol): @@ -22,7 +24,7 @@ def __getitem__(self, key: str) -> str: ... @overload - def get(self, key: str) -> str | None: + def get(self, key: str) -> 'str | None': ... @overload @@ -31,25 +33,30 @@ def get(self, key: str, default: str) -> str: class _HeadersDict(dict[str, str]): - def raw(self, key: str) -> bytes | None: + def raw(self, key: str) -> 'bytes | None': ... class _Request(Protocol): - query: _RequestParams - params: _RequestParams - headers: _HeadersDict - request: _Request + @property + def query(self) -> _RequestParams: + ... + @property + def params(self) -> _RequestParams: + ... -def parse_qs(qs: str) -> dict[str, str]: - return dict(bottle._parse_qsl(qs)) # type: ignore + @property + def headers(self) -> _HeadersDict: + ... + request = cast('_Request', None) -def require(cond: bool, message: str): - if not cond: - print(f'REQUIREMENT FAILED: {message}') - raise bottle.HTTPError(400, message) + +def parse_qs(qs: str) -> 'dict[str, str]': + # Re-use the bottle.py query string parser. It's a private function, but + # we're using a fixed version of Bottle. + return dict(bottle._parse_qsl(qs)) # type: ignore _HandlerFuncT = Callable[ @@ -58,6 +65,7 @@ def require(cond: bool, message: str): def handle_asserts(fn: _HandlerFuncT) -> _HandlerFuncT: + "Convert assertion failures into HTTP 400s" @functools.wraps(fn) def wrapped(): @@ -72,17 +80,10 @@ def wrapped(): return wrapped -def test_flags() -> dict[str, str]: +def test_params() -> 'dict[str, str]': return parse_qs(request.headers.get('X-MongoDB-HTTP-TestParams', '')) -def maybe_pause(): - pause = int(test_flags().get('pause', '0')) - if pause: - print(f'Pausing for {pause} seconds') - time.sleep(pause) - - @imds.get('/metadata/identity/oauth2/token') @handle_asserts def get_oauth2_token(): @@ -91,10 +92,7 @@ def get_oauth2_token(): resource = request.query['resource'] assert resource == 'https://vault.azure.net', 'Only https://vault.azure.net is supported' - flags = test_flags() - maybe_pause() - - case = flags.get('case') + case = test_params().get('case') print('Case is:', case) if case == '404': return HTTPResponse(status=404) @@ -114,17 +112,18 @@ def get_oauth2_token(): if case == 'slow': return _slow() - assert case is None or case == '', f'Unknown HTTP test case "{case}"' + assert case in (None, ''), 'Unknown HTTP test case "{}"'.format(case) return { 'access_token': 'magic-cookie', - 'expires_in': '60', + 'expires_in': '70', 'token_type': 'Bearer', 'resource': 'https://vault.azure.net', } def _gen_giant() -> Iterable[bytes]: + "Generate a giant message" yield b'{ "item": [' for _ in range(1024 * 256): yield (b'null, null, null, null, null, null, null, null, null, null, ' @@ -136,6 +135,7 @@ def _gen_giant() -> Iterable[bytes]: def _slow() -> Iterable[bytes]: + "Generate a very slow message" yield b'{ "item": [' for _ in range(1000): yield b'null, ' @@ -144,6 +144,8 @@ def _slow() -> Iterable[bytes]: if __name__ == '__main__': - print(f'RECOMMENDED: Run this script using bottle.py in the same ' - f'directory (e.g. [{sys.executable} bottle.py fake_azure:imds])') + print( + 'RECOMMENDED: Run this script using bottle.py (e.g. [{} {}/bottle.py fake_azure:imds])' + .format(sys.executable, + Path(__file__).resolve().parent)) imds.run() diff --git a/build/proc-ctl.py b/build/proc-ctl.py new file mode 100644 index 00000000000..9446bf2892f --- /dev/null +++ b/build/proc-ctl.py @@ -0,0 +1,300 @@ +""" +Extremely basic subprocess control +""" + +import argparse +import json +import os +import random +import signal +import subprocess +import sys +import time +import traceback +from datetime import datetime, timedelta +from pathlib import Path +from typing import TYPE_CHECKING, NoReturn, Sequence, Union, cast + +if TYPE_CHECKING: + from typing import (Literal, NamedTuple, TypedDict) + +INTERUPT_SIGNAL = signal.SIGINT if os.name != 'nt' else signal.CTRL_C_SIGNAL + + +def create_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser('proc-ctl') + grp = parser.add_subparsers(title='Commands', + dest='command', + metavar='') + + start = grp.add_parser('start', help='Start a new subprocess') + start.add_argument('--ctl-dir', + help='The control directory for the subprocess', + required=True, + type=Path) + start.add_argument('--cwd', + help='The new subdirectory of the spawned process', + type=Path) + start.add_argument( + '--spawn-wait', + help='Number of seconds to wait for child to be running', + type=float, + default=3) + start.add_argument('child_command', + nargs='+', + help='The command to execute', + metavar=' [args...]') + + stop = grp.add_parser('stop', help='Stop a running subprocess') + stop.add_argument('--ctl-dir', + help='The control directory for the subprocess', + required=True, + type=Path) + stop.add_argument('--stop-wait', + help='Number of seconds to wait for stopping', + type=float, + default=5) + stop.add_argument('--if-not-running', + help='Action to take if the child is not running', + choices=['fail', 'ignore'], + default='fail') + + ll_run = grp.add_parser('__run') + ll_run.add_argument('--ctl-dir', type=Path, required=True) + ll_run.add_argument('child_command', nargs='+') + + return parser + + +if TYPE_CHECKING: + StartCommandArgs = NamedTuple('StartCommandArgs', [ + ('command', Literal['start']), + ('ctl_dir', Path), + ('cwd', Path), + ('child_command', Sequence[str]), + ('spawn_wait', float), + ]) + + StopCommandArgs = NamedTuple('StopCommandArgs', [ + ('command', Literal['stop']), + ('ctl_dir', Path), + ('stop_wait', float), + ('if_not_running', Literal['fail', 'ignore']), + ]) + + _RunCommandArgs = NamedTuple('_RunCommandArgs', [ + ('command', Literal['__run']), + ('child_command', Sequence[str]), + ('ctl_dir', Path), + ]) + + CommandArgs = Union[StartCommandArgs, StopCommandArgs, _RunCommandArgs] + + _ResultType = TypedDict('_ResultType', { + 'exit': 'str | int | None', + 'error': 'str | None' + }) + + +def parse_argv(argv: 'Sequence[str]') -> 'CommandArgs': + parser = create_parser() + args = parser.parse_args(argv) + return cast('CommandArgs', args) + + +class _ChildControl: + + def __init__(self, ctl_dir: Path) -> None: + self._ctl_dir = ctl_dir + + @property + def pid_file(self): + """The file containing the child PID""" + return self._ctl_dir / 'pid.txt' + + @property + def result_file(self): + """The file containing the exit result""" + return self._ctl_dir / 'exit.json' + + def set_pid(self, pid: int): + write_text(self.pid_file, str(pid)) + + def get_pid(self) -> 'int | None': + try: + txt = self.pid_file.read_text() + except FileNotFoundError: + return None + return int(txt) + + def set_exit(self, exit: 'str | int | None', error: 'str | None') -> None: + write_text(self.result_file, json.dumps({ + 'exit': exit, + 'error': error + })) + remove_file(self.pid_file) + + def get_result(self) -> 'None | _ResultType': + try: + txt = self.result_file.read_text() + except FileNotFoundError: + return None + return json.loads(txt) + + def clear_result(self) -> None: + remove_file(self.result_file) + + +def _start(args: 'StartCommandArgs') -> int: + ll_run_cmd = [ + sys.executable, + '-u', + '--', + __file__, + '__run', + '--ctl-dir={}'.format(args.ctl_dir), + '--', + *args.child_command, + ] + args.ctl_dir.mkdir(exist_ok=True, parents=True) + child = _ChildControl(args.ctl_dir) + if child.get_pid() is not None: + raise RuntimeError('Child process is already running [PID {}]'.format( + child.get_pid())) + child.clear_result() + # Spawn the child controller + subprocess.Popen( + ll_run_cmd, + cwd=args.cwd, + stderr=subprocess.STDOUT, + stdout=args.ctl_dir.joinpath('runner-output.txt').open('wb'), + stdin=subprocess.DEVNULL) + expire = datetime.now() + timedelta(seconds=args.spawn_wait) + # Wait for the PID to appear + while child.get_pid() is None and child.get_result() is None: + if expire < datetime.now(): + break + time.sleep(0.1) + # Check that it actually spawned + if child.get_pid() is None: + result = child.get_result() + if result is None: + raise RuntimeError('Failed to spawn child runner?') + if result['error']: + print(result['error'], file=sys.stderr) + raise RuntimeError('Child exited immediately [Exited {}]'.format( + result['exit'])) + # Wait to see that it is still running after --spawn-wait seconds + while child.get_result() is None: + if expire < datetime.now(): + break + time.sleep(0.1) + # A final check to see if it is running + result = child.get_result() + if result is not None: + if result['error']: + print(result['error'], file=sys.stderr) + raise RuntimeError('Child exited prematurely [Exited {}]'.format( + result['exit'])) + return 0 + + +def _stop(args: 'StopCommandArgs') -> int: + child = _ChildControl(args.ctl_dir) + pid = child.get_pid() + if pid is None: + if args.if_not_running == 'fail': + raise RuntimeError('Child process is not running') + elif args.if_not_running == 'ignore': + # Nothing to do + return 0 + else: + assert False + os.kill(pid, INTERUPT_SIGNAL) + expire_at = datetime.now() + timedelta(seconds=args.stop_wait) + while expire_at > datetime.now() and child.get_result() is None: + time.sleep(0.1) + result = child.get_result() + if result is None: + raise RuntimeError( + 'Child process did not exit within the grace period') + return 0 + + +def __run(args: '_RunCommandArgs') -> int: + this = _ChildControl(args.ctl_dir) + try: + pipe = subprocess.Popen( + args.child_command, + stdout=args.ctl_dir.joinpath('child-output.txt').open('wb'), + stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL) + except: + this.set_exit('spawn-failed', traceback.format_exc()) + raise + this.set_pid(pipe.pid) + retc = None + try: + while 1: + try: + retc = pipe.wait(0.5) + except subprocess.TimeoutExpired: + pass + except KeyboardInterrupt: + pipe.send_signal(INTERUPT_SIGNAL) + if retc is not None: + break + finally: + this.set_exit(retc, None) + return 0 + + +def write_text(fpath: Path, content: str): + """ + "Atomically" write a new file. + + This writes the given ``content`` into a temporary file, then renames that + file into place. This prevents readers from seeing a partial read. + """ + tmp = fpath.with_name(fpath.name + '.tmp') + remove_file(tmp) + tmp.write_text(content) + os.sync() + remove_file(fpath) + tmp.rename(fpath) + + +def remove_file(fpath: Path): + """ + Safely remove a file. + + Because Win32, deletes are asynchronous, so we rename to a random filename, + then delete that file. This ensures the file is "out of the way", even if + it takes some time to delete. + """ + delname = fpath.with_name(fpath.name + '.delete-' + + str(random.randint(0, 999999))) + try: + fpath.rename(delname) + except FileNotFoundError: + return + delname.unlink() + + +def main(argv: 'Sequence[str]') -> int: + args = parse_argv(argv) + if args.command == 'start': + return _start(args) + if args.command == '__run': + return __run(args) + if args.command == 'stop': + return _stop(args) + return 0 + + +def start_main() -> NoReturn: + sys.exit(main(sys.argv[1:])) + + +if __name__ == '__main__': + start_main() diff --git a/src/libmongoc/tests/test-mcd-azure-imds.c b/src/libmongoc/tests/test-mcd-azure-imds.c index 842c1d530a5..70d85816316 100644 --- a/src/libmongoc/tests/test-mcd-azure-imds.c +++ b/src/libmongoc/tests/test-mcd-azure-imds.c @@ -101,16 +101,14 @@ _test_with_mock_server (void *ctx) _run_http_test_case ("", 0, 0, ""); // (No error) _run_http_test_case ("404", MONGOC_ERROR_AZURE, MONGOC_ERROR_AZURE_HTTP, ""); + _run_http_test_case ( + "slow", MONGOC_ERROR_STREAM, MONGOC_ERROR_STREAM_SOCKET, "Timeout"); _run_http_test_case ( "empty-json", MONGOC_ERROR_AZURE, MONGOC_ERROR_AZURE_BAD_JSON, ""); _run_http_test_case ( "bad-json", MONGOC_ERROR_CLIENT, MONGOC_ERROR_STREAM_INVALID_TYPE, ""); - _run_http_test_case ( "giant", MONGOC_ERROR_STREAM, MONGOC_ERROR_STREAM_SOCKET, "too large"); - - _run_http_test_case ( - "slow", MONGOC_ERROR_STREAM, MONGOC_ERROR_STREAM_SOCKET, "Timeout"); } static int