diff --git a/docker/mongodb-kubernetes-tests/Dockerfile b/docker/mongodb-kubernetes-tests/Dockerfile index fc30c4180..424f5ee76 100644 --- a/docker/mongodb-kubernetes-tests/Dockerfile +++ b/docker/mongodb-kubernetes-tests/Dockerfile @@ -43,6 +43,10 @@ RUN mkdir -p /tmp/mongodb-tools && \ cp /tmp/mongodb-tools/*/bin/* /usr/local/bin/ && \ rm -rf /tmp/mongodb-tools /tmp/mongodb-tools.tgz +RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" \ + && chmod +x ./kubectl \ + && mv ./kubectl /usr/local/bin/kubectl + COPY --from=builder /venv /venv ENV PATH="/venv/bin:${PATH}" diff --git a/docker/mongodb-kubernetes-tests/kubetester/helm.py b/docker/mongodb-kubernetes-tests/kubetester/helm.py index da0a19f39..da6887b81 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/helm.py +++ b/docker/mongodb-kubernetes-tests/kubetester/helm.py @@ -1,3 +1,4 @@ +import glob import logging import os import re @@ -120,8 +121,16 @@ def helm_repo_add(repo_name: str, url: str): def process_run_and_check(args, **kwargs): try: + logger.debug(f"subprocess.run: {args}") completed_process = subprocess.run(args, **kwargs) - completed_process.check_returncode() + # always print process output + if completed_process.stdout is not None: + stdout = completed_process.stdout.decode("utf-8") + logger.debug(f"stdout: {stdout}") + if completed_process.stderr is not None: + stderr = completed_process.stderr.decode("utf-8") + logger.debug(f"stderr: {stderr}") + completed_process.check_returncode() except subprocess.CalledProcessError as exc: if exc.stdout is not None: stdout = exc.stdout.decode("utf-8") @@ -141,10 +150,17 @@ def helm_upgrade( helm_options: Optional[List[str]] = None, helm_override_path: Optional[bool] = False, custom_operator_version: Optional[str] = None, + apply_crds_first: bool = False, ): if not helm_chart_path: logger.warning("Helm chart path is empty, defaulting to 'helm_chart'") helm_chart_path = "helm_chart" + + chart_dir = helm_chart_path if helm_override_path else _helm_chart_dir(helm_chart_path) + + if apply_crds_first: + apply_crds_from_chart(chart_dir) + command_args = _create_helm_args(helm_args, helm_options) args = [ "helm", @@ -154,19 +170,30 @@ def helm_upgrade( *command_args, name, ] + if custom_operator_version: args.append(f"--version={custom_operator_version}") - if helm_override_path: - args.append(helm_chart_path) - else: - args.append(_helm_chart_dir(helm_chart_path)) + + args.append(chart_dir) command = " ".join(args) - logger.debug("Running helm upgrade command:") - logger.debug(command) process_run_and_check(command, check=True, capture_output=True, shell=True) +def apply_crds_from_chart(chart_dir: str): + crd_files = glob.glob(os.path.join(chart_dir, "crds", "*.yaml")) + + if not crd_files: + raise Exception(f"No CRD files found in chart directory: {chart_dir}") + + logger.info(f"Found {len(crd_files)} CRD files to apply:") + + for crd_file in crd_files: + logger.info(f"Applying CRD from file: {crd_file}") + args = ["kubectl", "apply", "-f", crd_file] + process_run_and_check(" ".join(args), check=True, capture_output=True, shell=True) + + def helm_uninstall(name): args = ("helm", "uninstall", name) logger.info(args) diff --git a/docker/mongodb-kubernetes-tests/kubetester/operator.py b/docker/mongodb-kubernetes-tests/kubetester/operator.py index 08455c1d5..fa2b90344 100644 --- a/docker/mongodb-kubernetes-tests/kubetester/operator.py +++ b/docker/mongodb-kubernetes-tests/kubetester/operator.py @@ -95,7 +95,9 @@ def install(self, custom_operator_version: Optional[str] = None) -> Operator: return self - def upgrade(self, multi_cluster: bool = False, custom_operator_version: Optional[str] = None) -> Operator: + def upgrade( + self, multi_cluster: bool = False, custom_operator_version: Optional[str] = None, apply_crds_first: bool = False + ) -> Operator: """Upgrades the Operator in Kubernetes cluster using 'helm upgrade', waits until it's running""" helm_upgrade( self.name, @@ -104,6 +106,7 @@ def upgrade(self, multi_cluster: bool = False, custom_operator_version: Optional helm_chart_path=self.helm_chart_path, helm_options=self.helm_options, custom_operator_version=custom_operator_version, + apply_crds_first=apply_crds_first, ) self._wait_for_operator_ready() self._wait_operator_webhook_is_ready(multi_cluster=multi_cluster) diff --git a/docker/mongodb-kubernetes-tests/tests/conftest.py b/docker/mongodb-kubernetes-tests/tests/conftest.py index 9d4364d02..d9a3da265 100644 --- a/docker/mongodb-kubernetes-tests/tests/conftest.py +++ b/docker/mongodb-kubernetes-tests/tests/conftest.py @@ -107,7 +107,7 @@ def get_version_id(): @fixture(scope="module") -def operator_installation_config(namespace: str) -> Dict[str, str]: +def operator_installation_config(namespace: str) -> dict[str, str]: return get_operator_installation_config(namespace) @@ -126,7 +126,7 @@ def get_operator_installation_config(namespace): @fixture(scope="module") -def monitored_appdb_operator_installation_config(operator_installation_config: Dict[str, str]) -> Dict[str, str]: +def monitored_appdb_operator_installation_config(operator_installation_config: dict[str, str]) -> dict[str, str]: """Returns the ConfigMap containing configuration data for the Operator to be created and for the AppDB to be monitored. Created in the single_e2e.sh""" @@ -135,7 +135,7 @@ def monitored_appdb_operator_installation_config(operator_installation_config: D return config -def get_multi_cluster_operator_installation_config(namespace: str) -> Dict[str, str]: +def get_multi_cluster_operator_installation_config(namespace: str) -> dict[str, str]: """Returns the ConfigMap containing configuration data for the Operator to be created. Created in the single_e2e.sh""" config = KubernetesTester.read_configmap( @@ -150,7 +150,7 @@ def get_multi_cluster_operator_installation_config(namespace: str) -> Dict[str, @fixture(scope="module") def multi_cluster_operator_installation_config( central_cluster_client: kubernetes.client.ApiClient, namespace: str -) -> Dict[str, str]: +) -> dict[str, str]: return get_multi_cluster_operator_installation_config(namespace) @@ -159,7 +159,7 @@ def multi_cluster_monitored_appdb_operator_installation_config( central_cluster_client: kubernetes.client.ApiClient, namespace: str, multi_cluster_operator_installation_config: dict[str, str], -) -> Dict[str, str]: +) -> dict[str, str]: multi_cluster_operator_installation_config["customEnvVars"] = f"OPS_MANAGER_MONITOR_APPDB=true" return multi_cluster_operator_installation_config @@ -167,7 +167,7 @@ def multi_cluster_monitored_appdb_operator_installation_config( @fixture(scope="module") def operator_clusterwide( namespace: str, - operator_installation_config: Dict[str, str], + operator_installation_config: dict[str, str], ) -> Operator: return get_operator_clusterwide(namespace, operator_installation_config) @@ -181,7 +181,7 @@ def get_operator_clusterwide(namespace, operator_installation_config): @fixture(scope="module") def operator_vault_secret_backend( namespace: str, - monitored_appdb_operator_installation_config: Dict[str, str], + monitored_appdb_operator_installation_config: dict[str, str], ) -> Operator: helm_args = monitored_appdb_operator_installation_config.copy() helm_args["operator.vaultSecretBackend.enabled"] = "true" @@ -191,7 +191,7 @@ def operator_vault_secret_backend( @fixture(scope="module") def operator_vault_secret_backend_tls( namespace: str, - monitored_appdb_operator_installation_config: Dict[str, str], + monitored_appdb_operator_installation_config: dict[str, str], ) -> Operator: helm_args = monitored_appdb_operator_installation_config.copy() helm_args["operator.vaultSecretBackend.enabled"] = "true" @@ -200,7 +200,7 @@ def operator_vault_secret_backend_tls( @fixture(scope="module") -def operator_installation_config_quick_recovery(operator_installation_config: Dict[str, str]) -> Dict[str, str]: +def operator_installation_config_quick_recovery(operator_installation_config: dict[str, str]) -> dict[str, str]: """ This functions appends automatic recovery settings for CLOUDP-189433. In order to make the test runnable in reasonable time, we override the Recovery back off to 120 seconds. This gives enough time for the initial @@ -480,9 +480,9 @@ def get_custom_om_version(): @fixture(scope="module") def default_operator( namespace: str, - operator_installation_config: Dict[str, str], + operator_installation_config: dict[str, str], central_cluster_name: str, - multi_cluster_operator_installation_config: Dict[str, str], + multi_cluster_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], member_cluster_names: List[str], @@ -500,8 +500,7 @@ def default_operator( def get_default_operator( - namespace: str, - operator_installation_config: Dict[str, str], + namespace: str, operator_installation_config: dict[str, str], apply_crds_first: bool = False ) -> Operator: """Installs/upgrades a default Operator used by any test not interested in some custom Operator setting. TODO we use the helm template | kubectl apply -f process so far as Helm install/upgrade needs more refactoring in @@ -509,7 +508,7 @@ def get_default_operator( operator = Operator( namespace=namespace, helm_args=operator_installation_config, - ).upgrade() + ).upgrade(apply_crds_first=apply_crds_first) return operator @@ -517,7 +516,7 @@ def get_default_operator( @fixture(scope="module") def operator_with_monitored_appdb( namespace: str, - monitored_appdb_operator_installation_config: Dict[str, str], + monitored_appdb_operator_installation_config: dict[str, str], ) -> Operator: """Installs/upgrades a default Operator used by any test that needs the AppDB monitoring enabled.""" return Operator( @@ -628,7 +627,7 @@ def member_cluster_clients() -> List[MultiClusterClient]: def multi_cluster_operator( namespace: str, central_cluster_name: str, - multi_cluster_operator_installation_config: Dict[str, str], + multi_cluster_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], member_cluster_names: List[str], @@ -646,10 +645,11 @@ def multi_cluster_operator( def get_multi_cluster_operator( namespace: str, central_cluster_name: str, - multi_cluster_operator_installation_config: Dict[str, str], + multi_cluster_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], member_cluster_names: List[str], + apply_crds_first: bool = False, ) -> Operator: os.environ["HELM_KUBECONTEXT"] = central_cluster_name @@ -667,6 +667,7 @@ def get_multi_cluster_operator( "operator.createOperatorServiceAccount": "false", }, central_cluster_name, + apply_crds_first=apply_crds_first, ) @@ -674,7 +675,7 @@ def get_multi_cluster_operator( def multi_cluster_operator_with_monitored_appdb( namespace: str, central_cluster_name: str, - multi_cluster_monitored_appdb_operator_installation_config: Dict[str, str], + multi_cluster_monitored_appdb_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], member_cluster_names: List[str], @@ -703,7 +704,7 @@ def multi_cluster_operator_with_monitored_appdb( def multi_cluster_operator_manual_remediation( namespace: str, central_cluster_name: str, - multi_cluster_operator_installation_config: Dict[str, str], + multi_cluster_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], member_cluster_names: List[str], @@ -754,7 +755,7 @@ def get_multi_cluster_operator_clustermode(namespace: str) -> Operator: def multi_cluster_operator_clustermode( namespace: str, central_cluster_name: str, - multi_cluster_operator_installation_config: Dict[str, str], + multi_cluster_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], member_cluster_names: List[str], @@ -767,7 +768,7 @@ def multi_cluster_operator_clustermode( def install_multi_cluster_operator_set_members_fn( namespace: str, central_cluster_name: str, - multi_cluster_operator_installation_config: Dict[str, str], + multi_cluster_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], ) -> Callable[[List[str]], Operator]: @@ -793,14 +794,15 @@ def _fn(member_cluster_names: List[str]) -> Operator: def _install_multi_cluster_operator( namespace: str, - multi_cluster_operator_installation_config: Dict[str, str], + multi_cluster_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], - helm_opts: Dict[str, str], + helm_opts: dict[str, str], central_cluster_name: str, operator_name: Optional[str] = MULTI_CLUSTER_OPERATOR_NAME, helm_chart_path: Optional[str] = LOCAL_HELM_CHART_DIR, custom_operator_version: Optional[str] = None, + apply_crds_first: bool = False, ) -> Operator: multi_cluster_operator_installation_config.update(helm_opts) @@ -822,7 +824,7 @@ def _install_multi_cluster_operator( helm_args=multi_cluster_operator_installation_config, api_client=central_cluster_client, helm_chart_path=helm_chart_path, - ).upgrade(multi_cluster=True, custom_operator_version=custom_operator_version) + ).upgrade(multi_cluster=True, custom_operator_version=custom_operator_version, apply_crds_first=apply_crds_first) # If we're running locally, then immediately after installing the deployment, we scale it to zero. # This way operator in POD is not interfering with locally running one. @@ -840,7 +842,7 @@ def _install_multi_cluster_operator( def official_operator( namespace: str, managed_security_context: str, - operator_installation_config: Dict[str, str], + operator_installation_config: dict[str, str], central_cluster_name: str, central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], @@ -919,7 +921,7 @@ def install_legacy_deployment_state_meko( def install_official_operator( namespace: str, managed_security_context: str, - operator_installation_config: Dict[str, str], + operator_installation_config: dict[str, str], central_cluster_name: Optional[str], central_cluster_client: Optional[client.ApiClient], member_cluster_clients: Optional[List[MultiClusterClient]], @@ -1033,7 +1035,7 @@ def log_deployment_and_images(deployments): # Extract container images and deployments names from the nested dict returned by kubetester # Handles any missing key gracefully -def extract_container_images_and_deployments(deployments) -> (Dict[str, str], List[str]): +def extract_container_images_and_deployments(deployments) -> (dict[str, str], List[str]): deployment_images = {} deployment_names = [] deployments = deployments.to_dict() @@ -1318,7 +1320,7 @@ def get_api_servers_from_kubeconfig_secret( return get_api_servers_from_pod_kubeconfig(kubeconfig_secret["kubeconfig"], cluster_clients) -def get_api_servers_from_test_pod_kubeconfig(namespace: str, member_cluster_names: List[str]) -> Dict[str, str]: +def get_api_servers_from_test_pod_kubeconfig(namespace: str, member_cluster_names: List[str]) -> dict[str, str]: test_pod_cluster = get_test_pod_cluster_name() cluster_clients = get_clients_for_clusters(member_cluster_names) diff --git a/docker/mongodb-kubernetes-tests/tests/upgrades/appdb_tls_operator_upgrade_v1_32_to_mck.py b/docker/mongodb-kubernetes-tests/tests/upgrades/appdb_tls_operator_upgrade_v1_32_to_mck.py index 5d01cb1da..479284743 100644 --- a/docker/mongodb-kubernetes-tests/tests/upgrades/appdb_tls_operator_upgrade_v1_32_to_mck.py +++ b/docker/mongodb-kubernetes-tests/tests/upgrades/appdb_tls_operator_upgrade_v1_32_to_mck.py @@ -10,7 +10,10 @@ LEGACY_OPERATOR_CHART, LEGACY_OPERATOR_IMAGE_NAME, LEGACY_OPERATOR_NAME, + MULTI_CLUSTER_OPERATOR_NAME, create_appdb_certs, + get_default_operator, + get_multi_cluster_operator, install_official_operator, is_multi_cluster, ) @@ -127,8 +130,30 @@ def test_downscale_latest_official_operator(namespace: str): @mark.e2e_appdb_tls_operator_upgrade_v1_32_to_mck -def test_upgrade_operator(default_operator: Operator): - default_operator.assert_is_running() +def test_upgrade_operator( + namespace: str, + central_cluster_name: str, + multi_cluster_operator_installation_config: dict[str, str], + operator_installation_config: dict[str, str], + central_cluster_client, + member_cluster_clients, + member_cluster_names, +): + if is_multi_cluster(): + operator = get_multi_cluster_operator( + namespace, + central_cluster_name, + multi_cluster_operator_installation_config, + central_cluster_client, + member_cluster_clients, + member_cluster_names, + apply_crds_first=True, + ) + else: + operator = get_default_operator( + namespace, operator_installation_config=operator_installation_config, apply_crds_first=True + ) + operator.assert_is_running() @mark.e2e_appdb_tls_operator_upgrade_v1_32_to_mck diff --git a/docker/mongodb-kubernetes-tests/tests/upgrades/operator_upgrade_ops_manager.py b/docker/mongodb-kubernetes-tests/tests/upgrades/operator_upgrade_ops_manager.py index 0b887299c..532af4a2a 100644 --- a/docker/mongodb-kubernetes-tests/tests/upgrades/operator_upgrade_ops_manager.py +++ b/docker/mongodb-kubernetes-tests/tests/upgrades/operator_upgrade_ops_manager.py @@ -9,8 +9,7 @@ from kubetester.operator import Operator from kubetester.opsmanager import MongoDBOpsManager from pytest import fixture, mark -from tests.conftest import LEGACY_OPERATOR_NAME -from tests.upgrades import downscale_operator_deployment +from tests.conftest import get_default_operator, operator_installation_config @fixture(scope="module") @@ -144,8 +143,11 @@ def test_mdb_created(some_mdb: MongoDB): @mark.e2e_operator_upgrade_ops_manager -def test_upgrade_operator(default_operator: Operator): - default_operator.assert_is_running() +def test_upgrade_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator( + namespace, operator_installation_config=operator_installation_config, apply_crds_first=True + ) + operator.assert_is_running() @mark.e2e_operator_upgrade_ops_manager diff --git a/docker/mongodb-kubernetes-tests/tests/upgrades/operator_upgrade_replica_set.py b/docker/mongodb-kubernetes-tests/tests/upgrades/operator_upgrade_replica_set.py index 68a4cd513..121291fa0 100644 --- a/docker/mongodb-kubernetes-tests/tests/upgrades/operator_upgrade_replica_set.py +++ b/docker/mongodb-kubernetes-tests/tests/upgrades/operator_upgrade_replica_set.py @@ -1,5 +1,3 @@ -from kubernetes import client -from kubernetes.client import ApiException from kubetester import MongoDB from kubetester.certs import create_mongodb_tls_certs from kubetester.kubetester import KubernetesTester @@ -9,8 +7,7 @@ from kubetester.operator import Operator from pytest import fixture, mark from tests import test_logger -from tests.conftest import LEGACY_OPERATOR_NAME, log_deployments_info -from tests.upgrades import downscale_operator_deployment +from tests.conftest import get_default_operator RS_NAME = "my-replica-set" USER_PASSWORD = "/qwerty@!#:" @@ -97,8 +94,11 @@ def test_replicaset_user_created(replica_set_user: MongoDBUser): @mark.e2e_operator_upgrade_replica_set -def test_upgrade_operator(default_operator: Operator): - default_operator.assert_is_running() +def test_upgrade_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator( + namespace, operator_installation_config=operator_installation_config, apply_crds_first=True + ) + operator.assert_is_running() @mark.e2e_operator_upgrade_replica_set diff --git a/docker/mongodb-kubernetes-tests/tests/upgrades/sharded_cluster_operator_upgrade_v1_27_to_mck.py b/docker/mongodb-kubernetes-tests/tests/upgrades/sharded_cluster_operator_upgrade_v1_27_to_mck.py index 303b13bab..9e98bdce7 100644 --- a/docker/mongodb-kubernetes-tests/tests/upgrades/sharded_cluster_operator_upgrade_v1_27_to_mck.py +++ b/docker/mongodb-kubernetes-tests/tests/upgrades/sharded_cluster_operator_upgrade_v1_27_to_mck.py @@ -11,6 +11,7 @@ from tests.conftest import ( LEGACY_OPERATOR_NAME, OPERATOR_NAME, + get_default_operator, install_legacy_deployment_state_meko, log_deployments_info, ) @@ -113,9 +114,12 @@ def test_downscale_latest_official_operator(self, namespace: str): # and replacing it with MCK downscale_operator_deployment(deployment_name=LEGACY_OPERATOR_NAME, namespace=namespace) - def test_upgrade_operator(self, default_operator: Operator, namespace: str): + def test_upgrade_operator(self, namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator( + namespace, operator_installation_config=operator_installation_config, apply_crds_first=True + ) + operator.assert_is_running() logger.info("Installing the operator built from master") - default_operator.assert_is_running() # Dumping deployments in logs ensures we are using the correct operator version log_deployments_info(namespace) diff --git a/scripts/dev/prepare_local_e2e_run.sh b/scripts/dev/prepare_local_e2e_run.sh index 3f2d7d15c..58d6de19d 100755 --- a/scripts/dev/prepare_local_e2e_run.sh +++ b/scripts/dev/prepare_local_e2e_run.sh @@ -76,7 +76,7 @@ if [[ "${DEPLOY_OPERATOR:-"false"}" == "true" ]]; then fi # shellcheck disable=SC2128 - helm upgrade --install mongodb-enterprise-operator helm_chart --set "$(echo "${helm_values}" | tr ' ' ',')" + helm upgrade --install mongodb-kubernetes-operator helm_chart --set "$(echo "${helm_values}" | tr ' ' ',')" fi if [[ "${KUBE_ENVIRONMENT_NAME}" == "kind" ]]; then