From 385489285956c1d537430de7a9832a52c38f6147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Sierant?= Date: Wed, 30 Apr 2025 13:02:11 +0200 Subject: [PATCH 1/3] Install CRDs in operator upgrade tests --- .../kubetester/helm.py | 41 ++++++++++--- .../kubetester/operator.py | 5 +- .../tests/conftest.py | 58 ++++++++++--------- .../tests/shardedcluster/conftest.py | 4 +- .../upgrades/operator_upgrade_appdb_tls.py | 8 ++- .../upgrades/operator_upgrade_ops_manager.py | 8 ++- .../upgrades/operator_upgrade_replica_set.py | 8 ++- .../operator_upgrade_sharded_cluster.py | 8 ++- 8 files changed, 94 insertions(+), 46 deletions(-) diff --git a/docker/mongodb-enterprise-tests/kubetester/helm.py b/docker/mongodb-enterprise-tests/kubetester/helm.py index 99bf69cbd..eb4021759 100644 --- a/docker/mongodb-enterprise-tests/kubetester/helm.py +++ b/docker/mongodb-enterprise-tests/kubetester/helm.py @@ -1,3 +1,4 @@ +import glob import logging import os import re @@ -120,8 +121,16 @@ def helm_repo_add(repo_name: str, url: str): def process_run_and_check(args, **kwargs): try: + logger.debug(f"subprocess.run: {args}") completed_process = subprocess.run(args, **kwargs) - completed_process.check_returncode() + # always print process output + if completed_process.stdout is not None: + stdout = completed_process.stdout.decode("utf-8") + logger.debug(f"stdout: {stdout}") + if completed_process.stderr is not None: + stderr = completed_process.stderr.decode("utf-8") + logger.debug(f"stderr: {stderr}") + completed_process.check_returncode() except subprocess.CalledProcessError as exc: if exc.stdout is not None: stdout = exc.stdout.decode("utf-8") @@ -141,10 +150,17 @@ def helm_upgrade( helm_options: Optional[List[str]] = None, helm_override_path: Optional[bool] = False, custom_operator_version: Optional[str] = None, + apply_crds_first: bool = False, ): if not helm_chart_path: logger.warning("Helm chart path is empty, defaulting to 'helm_chart'") helm_chart_path = "helm_chart" + + chart_dir = helm_chart_path if helm_override_path else _helm_chart_dir(helm_chart_path) + + if apply_crds_first: + apply_crds_from_chart(chart_dir) + command_args = _create_helm_args(helm_args, helm_options) args = [ "helm", @@ -154,19 +170,30 @@ def helm_upgrade( *command_args, name, ] + if custom_operator_version: args.append(f"--version={custom_operator_version}") - if helm_override_path: - args.append(helm_chart_path) - else: - args.append(_helm_chart_dir(helm_chart_path)) + + args.append(chart_dir) command = " ".join(args) - logger.debug("Running helm upgrade command:") - logger.debug(command) process_run_and_check(command, check=True, capture_output=True, shell=True) +def apply_crds_from_chart(chart_dir: str): + crd_files = glob.glob(os.path.join(chart_dir, "crds", "*.yaml")) + + if not crd_files: + raise Exception(f"No CRD files found in chart directory: {chart_dir}") + + logger.info(f"Found {len(crd_files)} CRD files to apply:") + + for crd_file in crd_files: + logger.info(f"Applying CRD from file: {crd_file}") + args = ["kubectl", "apply", "-f", crd_file] + process_run_and_check(args, check=True, capture_output=True, shell=True) + + def helm_uninstall(name): args = ("helm", "uninstall", name) logger.info(args) diff --git a/docker/mongodb-enterprise-tests/kubetester/operator.py b/docker/mongodb-enterprise-tests/kubetester/operator.py index ee9d362d6..244e1fe7a 100644 --- a/docker/mongodb-enterprise-tests/kubetester/operator.py +++ b/docker/mongodb-enterprise-tests/kubetester/operator.py @@ -95,7 +95,9 @@ def install(self, custom_operator_version: Optional[str] = None) -> Operator: return self - def upgrade(self, multi_cluster: bool = False, custom_operator_version: Optional[str] = None) -> Operator: + def upgrade( + self, multi_cluster: bool = False, custom_operator_version: Optional[str] = None, apply_crds_first: bool = False + ) -> Operator: """Upgrades the Operator in Kubernetes cluster using 'helm upgrade', waits until it's running""" helm_upgrade( self.name, @@ -104,6 +106,7 @@ def upgrade(self, multi_cluster: bool = False, custom_operator_version: Optional helm_chart_path=self.helm_chart_path, helm_options=self.helm_options, custom_operator_version=custom_operator_version, + apply_crds_first=apply_crds_first, ) self._wait_for_operator_ready() self._wait_operator_webhook_is_ready(multi_cluster=multi_cluster) diff --git a/docker/mongodb-enterprise-tests/tests/conftest.py b/docker/mongodb-enterprise-tests/tests/conftest.py index 5bccdb9c8..8a13852b7 100644 --- a/docker/mongodb-enterprise-tests/tests/conftest.py +++ b/docker/mongodb-enterprise-tests/tests/conftest.py @@ -88,7 +88,7 @@ def get_version_id(): @fixture(scope="module") -def operator_installation_config(namespace: str) -> Dict[str, str]: +def operator_installation_config(namespace: str) -> dict[str, str]: return get_operator_installation_config(namespace) @@ -107,7 +107,7 @@ def get_operator_installation_config(namespace): @fixture(scope="module") -def monitored_appdb_operator_installation_config(operator_installation_config: Dict[str, str]) -> Dict[str, str]: +def monitored_appdb_operator_installation_config(operator_installation_config: dict[str, str]) -> dict[str, str]: """Returns the ConfigMap containing configuration data for the Operator to be created and for the AppDB to be monitored. Created in the single_e2e.sh""" @@ -116,7 +116,7 @@ def monitored_appdb_operator_installation_config(operator_installation_config: D return config -def get_multi_cluster_operator_installation_config(namespace: str) -> Dict[str, str]: +def get_multi_cluster_operator_installation_config(namespace: str) -> dict[str, str]: """Returns the ConfigMap containing configuration data for the Operator to be created. Created in the single_e2e.sh""" config = KubernetesTester.read_configmap( @@ -131,7 +131,7 @@ def get_multi_cluster_operator_installation_config(namespace: str) -> Dict[str, @fixture(scope="module") def multi_cluster_operator_installation_config( central_cluster_client: kubernetes.client.ApiClient, namespace: str -) -> Dict[str, str]: +) -> dict[str, str]: return get_multi_cluster_operator_installation_config(namespace) @@ -140,7 +140,7 @@ def multi_cluster_monitored_appdb_operator_installation_config( central_cluster_client: kubernetes.client.ApiClient, namespace: str, multi_cluster_operator_installation_config: dict[str, str], -) -> Dict[str, str]: +) -> dict[str, str]: multi_cluster_operator_installation_config["customEnvVars"] = f"OPS_MANAGER_MONITOR_APPDB=true" return multi_cluster_operator_installation_config @@ -148,7 +148,7 @@ def multi_cluster_monitored_appdb_operator_installation_config( @fixture(scope="module") def operator_clusterwide( namespace: str, - operator_installation_config: Dict[str, str], + operator_installation_config: dict[str, str], ) -> Operator: return get_operator_clusterwide(namespace, operator_installation_config) @@ -162,7 +162,7 @@ def get_operator_clusterwide(namespace, operator_installation_config): @fixture(scope="module") def operator_vault_secret_backend( namespace: str, - monitored_appdb_operator_installation_config: Dict[str, str], + monitored_appdb_operator_installation_config: dict[str, str], ) -> Operator: helm_args = monitored_appdb_operator_installation_config.copy() helm_args["operator.vaultSecretBackend.enabled"] = "true" @@ -172,7 +172,7 @@ def operator_vault_secret_backend( @fixture(scope="module") def operator_vault_secret_backend_tls( namespace: str, - monitored_appdb_operator_installation_config: Dict[str, str], + monitored_appdb_operator_installation_config: dict[str, str], ) -> Operator: helm_args = monitored_appdb_operator_installation_config.copy() helm_args["operator.vaultSecretBackend.enabled"] = "true" @@ -181,7 +181,7 @@ def operator_vault_secret_backend_tls( @fixture(scope="module") -def operator_installation_config_quick_recovery(operator_installation_config: Dict[str, str]) -> Dict[str, str]: +def operator_installation_config_quick_recovery(operator_installation_config: dict[str, str]) -> dict[str, str]: """ This functions appends automatic recovery settings for CLOUDP-189433. In order to make the test runnable in reasonable time, we override the Recovery back off to 120 seconds. This gives enough time for the initial @@ -461,9 +461,9 @@ def get_custom_om_version(): @fixture(scope="module") def default_operator( namespace: str, - operator_installation_config: Dict[str, str], + operator_installation_config: dict[str, str], central_cluster_name: str, - multi_cluster_operator_installation_config: Dict[str, str], + multi_cluster_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], member_cluster_names: List[str], @@ -491,8 +491,7 @@ def community_operator(namespace: str) -> Operator: def get_default_operator( - namespace: str, - operator_installation_config: Dict[str, str], + namespace: str, operator_installation_config: dict[str, str], apply_crds_first: bool = False ) -> Operator: """Installs/upgrades a default Operator used by any test not interested in some custom Operator setting. TODO we use the helm template | kubectl apply -f process so far as Helm install/upgrade needs more refactoring in @@ -500,7 +499,7 @@ def get_default_operator( operator = Operator( namespace=namespace, helm_args=operator_installation_config, - ).upgrade() + ).upgrade(apply_crds_first=apply_crds_first) return operator @@ -508,7 +507,7 @@ def get_default_operator( @fixture(scope="module") def operator_with_monitored_appdb( namespace: str, - monitored_appdb_operator_installation_config: Dict[str, str], + monitored_appdb_operator_installation_config: dict[str, str], ) -> Operator: """Installs/upgrades a default Operator used by any test that needs the AppDB monitoring enabled.""" return Operator( @@ -619,7 +618,7 @@ def member_cluster_clients() -> List[MultiClusterClient]: def multi_cluster_operator( namespace: str, central_cluster_name: str, - multi_cluster_operator_installation_config: Dict[str, str], + multi_cluster_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], member_cluster_names: List[str], @@ -637,10 +636,11 @@ def multi_cluster_operator( def get_multi_cluster_operator( namespace: str, central_cluster_name: str, - multi_cluster_operator_installation_config: Dict[str, str], + multi_cluster_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], member_cluster_names: List[str], + apply_crds_first: bool = False, ) -> Operator: os.environ["HELM_KUBECONTEXT"] = central_cluster_name @@ -658,6 +658,7 @@ def get_multi_cluster_operator( "operator.createOperatorServiceAccount": "false", }, central_cluster_name, + apply_crds_first=apply_crds_first, ) @@ -665,7 +666,7 @@ def get_multi_cluster_operator( def multi_cluster_operator_with_monitored_appdb( namespace: str, central_cluster_name: str, - multi_cluster_monitored_appdb_operator_installation_config: Dict[str, str], + multi_cluster_monitored_appdb_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], member_cluster_names: List[str], @@ -694,7 +695,7 @@ def multi_cluster_operator_with_monitored_appdb( def multi_cluster_operator_manual_remediation( namespace: str, central_cluster_name: str, - multi_cluster_operator_installation_config: Dict[str, str], + multi_cluster_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], member_cluster_names: List[str], @@ -745,7 +746,7 @@ def get_multi_cluster_operator_clustermode(namespace: str) -> Operator: def multi_cluster_operator_clustermode( namespace: str, central_cluster_name: str, - multi_cluster_operator_installation_config: Dict[str, str], + multi_cluster_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], member_cluster_names: List[str], @@ -758,7 +759,7 @@ def multi_cluster_operator_clustermode( def install_multi_cluster_operator_set_members_fn( namespace: str, central_cluster_name: str, - multi_cluster_operator_installation_config: Dict[str, str], + multi_cluster_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], ) -> Callable[[List[str]], Operator]: @@ -784,14 +785,15 @@ def _fn(member_cluster_names: List[str]) -> Operator: def _install_multi_cluster_operator( namespace: str, - multi_cluster_operator_installation_config: Dict[str, str], + multi_cluster_operator_installation_config: dict[str, str], central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], - helm_opts: Dict[str, str], + helm_opts: dict[str, str], central_cluster_name: str, operator_name: Optional[str] = MULTI_CLUSTER_OPERATOR_NAME, helm_chart_path: Optional[str] = "helm_chart", custom_operator_version: Optional[str] = None, + apply_crds_first: bool = False, ) -> Operator: prepare_multi_cluster_namespaces( namespace, @@ -808,7 +810,7 @@ def _install_multi_cluster_operator( helm_args=multi_cluster_operator_installation_config, api_client=central_cluster_client, helm_chart_path=helm_chart_path, - ).upgrade(multi_cluster=True, custom_operator_version=custom_operator_version) + ).upgrade(multi_cluster=True, custom_operator_version=custom_operator_version, apply_crds_first=apply_crds_first) # If we're running locally, then immediately after installing the deployment, we scale it to zero. # This way operator in POD is not interfering with locally running one. @@ -826,7 +828,7 @@ def _install_multi_cluster_operator( def official_operator( namespace: str, managed_security_context: str, - operator_installation_config: Dict[str, str], + operator_installation_config: dict[str, str], central_cluster_name: str, central_cluster_client: client.ApiClient, member_cluster_clients: List[MultiClusterClient], @@ -847,7 +849,7 @@ def official_operator( def install_official_operator( namespace: str, managed_security_context: str, - operator_installation_config: Dict[str, str], + operator_installation_config: dict[str, str], central_cluster_name: Optional[str], central_cluster_client: Optional[client.ApiClient], member_cluster_clients: Optional[List[MultiClusterClient]], @@ -950,7 +952,7 @@ def log_deployment_and_images(deployments): # Extract container images and deployments names from the nested dict returned by kubetester # Handles any missing key gracefully -def extract_container_images_and_deployments(deployments) -> (Dict[str, str], List[str]): +def extract_container_images_and_deployments(deployments) -> (dict[str, str], List[str]): deployment_images = {} deployment_names = [] deployments = deployments.to_dict() @@ -1232,7 +1234,7 @@ def get_api_servers_from_kubeconfig_secret( return get_api_servers_from_pod_kubeconfig(kubeconfig_secret["kubeconfig"], cluster_clients) -def get_api_servers_from_test_pod_kubeconfig(namespace: str, member_cluster_names: List[str]) -> Dict[str, str]: +def get_api_servers_from_test_pod_kubeconfig(namespace: str, member_cluster_names: List[str]) -> dict[str, str]: test_pod_cluster = get_test_pod_cluster_name() cluster_clients = get_clients_for_clusters(member_cluster_names) diff --git a/docker/mongodb-enterprise-tests/tests/shardedcluster/conftest.py b/docker/mongodb-enterprise-tests/tests/shardedcluster/conftest.py index 116070e79..255cafa56 100644 --- a/docker/mongodb-enterprise-tests/tests/shardedcluster/conftest.py +++ b/docker/mongodb-enterprise-tests/tests/shardedcluster/conftest.py @@ -3,7 +3,7 @@ from typing import Any, List import kubernetes -from _pytest.fixtures import fixture +import pytest from kubetester import MongoDB, read_configmap from kubetester.mongodb_multi import MultiClusterClient from kubetester.operator import Operator @@ -22,7 +22,7 @@ from tests.multicluster.conftest import cluster_spec_list -@fixture(scope="module") +@pytest.fixture(scope="module") def operator(namespace: str) -> Operator: if is_multi_cluster(): return get_multi_cluster_operator( diff --git a/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_appdb_tls.py b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_appdb_tls.py index 80c56024f..dfcbe3629 100644 --- a/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_appdb_tls.py +++ b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_appdb_tls.py @@ -7,6 +7,7 @@ from pytest import fixture, mark from tests.conftest import ( create_appdb_certs, + get_default_operator, install_official_operator, is_multi_cluster, ) @@ -110,8 +111,11 @@ def test_create_om_non_tls(ops_manager_non_tls: MongoDBOpsManager): @mark.e2e_operator_upgrade_appdb_tls -def test_upgrade_operator(default_operator: Operator): - default_operator.assert_is_running() +def test_upgrade_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator( + namespace, operator_installation_config=operator_installation_config, apply_crds_first=True + ) + operator.assert_is_running() @mark.e2e_operator_upgrade_appdb_tls diff --git a/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_ops_manager.py b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_ops_manager.py index e3c1e151f..532af4a2a 100644 --- a/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_ops_manager.py +++ b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_ops_manager.py @@ -9,6 +9,7 @@ from kubetester.operator import Operator from kubetester.opsmanager import MongoDBOpsManager from pytest import fixture, mark +from tests.conftest import get_default_operator, operator_installation_config @fixture(scope="module") @@ -142,8 +143,11 @@ def test_mdb_created(some_mdb: MongoDB): @mark.e2e_operator_upgrade_ops_manager -def test_upgrade_operator(default_operator: Operator): - default_operator.assert_is_running() +def test_upgrade_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator( + namespace, operator_installation_config=operator_installation_config, apply_crds_first=True + ) + operator.assert_is_running() @mark.e2e_operator_upgrade_ops_manager diff --git a/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_replica_set.py b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_replica_set.py index f8fe9be20..1a37dc990 100644 --- a/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_replica_set.py +++ b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_replica_set.py @@ -7,6 +7,7 @@ from kubetester.mongodb_user import MongoDBUser from kubetester.operator import Operator from pytest import fixture +from tests.conftest import get_default_operator RS_NAME = "my-replica-set" USER_PASSWORD = "/qwerty@!#:" @@ -91,8 +92,11 @@ def test_replicaset_user_created(replica_set_user: MongoDBUser): @pytest.mark.e2e_operator_upgrade_replica_set -def test_upgrade_operator(default_operator: Operator): - default_operator.assert_is_running() +def test_upgrade_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator( + namespace, operator_installation_config=operator_installation_config, apply_crds_first=True + ) + operator.assert_is_running() @pytest.mark.e2e_operator_upgrade_replica_set diff --git a/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_sharded_cluster.py b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_sharded_cluster.py index 9300d8d72..531721814 100644 --- a/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_sharded_cluster.py +++ b/docker/mongodb-enterprise-tests/tests/upgrades/operator_upgrade_sharded_cluster.py @@ -10,6 +10,7 @@ from tests import test_logger from tests.conftest import ( LEGACY_DEPLOYMENT_STATE_VERSION, + get_default_operator, install_official_operator, log_deployments_info, ) @@ -119,9 +120,12 @@ def test_scale_up_sharded_cluster(self, sharded_cluster: MongoDB): @pytest.mark.e2e_operator_upgrade_sharded_cluster class TestOperatorUpgrade: - def test_upgrade_operator(self, default_operator: Operator, namespace: str): + def test_upgrade_operator(self, namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator( + namespace, operator_installation_config=operator_installation_config, apply_crds_first=True + ) + operator.assert_is_running() logger.info("Installing the operator built from master") - default_operator.assert_is_running() # Dumping deployments in logs ensures we are using the correct operator version log_deployments_info(namespace) From bb976bad4de6bd88dc3455fa5f5049b7a09d43cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Sierant?= Date: Wed, 30 Apr 2025 14:51:17 +0200 Subject: [PATCH 2/3] Add kubectl download to test_app dockerfile --- docker/mongodb-enterprise-tests/Dockerfile | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docker/mongodb-enterprise-tests/Dockerfile b/docker/mongodb-enterprise-tests/Dockerfile index 078b5db5c..595a736e1 100644 --- a/docker/mongodb-enterprise-tests/Dockerfile +++ b/docker/mongodb-enterprise-tests/Dockerfile @@ -37,6 +37,19 @@ RUN curl --fail --retry 3 -L -o "${HELM_NAME}" "https://get.helm.sh/${HELM_NAME} && rm "${HELM_NAME}" \ && mv "linux-amd64/helm" "/usr/local/bin/helm" +<<<<<<< Updated upstream +======= +ADD https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2204-x86_64-100.12.0.tgz /tmp/mongodb-tools.tgz +RUN mkdir -p /tmp/mongodb-tools && \ + tar xfz /tmp/mongodb-tools.tgz -C /tmp/mongodb-tools && \ + cp /tmp/mongodb-tools/*/bin/* /usr/local/bin/ && \ + rm -rf /tmp/mongodb-tools /tmp/mongodb-tools.tgz + +RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" \ + && chmod +x ./kubectl \ + && mv ./kubectl /usr/local/bin/kubectl + +>>>>>>> Stashed changes COPY --from=builder /venv /venv ENV PATH="/venv/bin:${PATH}" From 25ba545ed721314f2250d3fd981656f955de73e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Sierant?= Date: Wed, 30 Apr 2025 15:41:19 +0200 Subject: [PATCH 3/3] Fixed dockerfile --- docker/mongodb-enterprise-tests/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/mongodb-enterprise-tests/Dockerfile b/docker/mongodb-enterprise-tests/Dockerfile index 595a736e1..424f5ee76 100644 --- a/docker/mongodb-enterprise-tests/Dockerfile +++ b/docker/mongodb-enterprise-tests/Dockerfile @@ -37,8 +37,6 @@ RUN curl --fail --retry 3 -L -o "${HELM_NAME}" "https://get.helm.sh/${HELM_NAME} && rm "${HELM_NAME}" \ && mv "linux-amd64/helm" "/usr/local/bin/helm" -<<<<<<< Updated upstream -======= ADD https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2204-x86_64-100.12.0.tgz /tmp/mongodb-tools.tgz RUN mkdir -p /tmp/mongodb-tools && \ tar xfz /tmp/mongodb-tools.tgz -C /tmp/mongodb-tools && \ @@ -49,7 +47,6 @@ RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl - && chmod +x ./kubectl \ && mv ./kubectl /usr/local/bin/kubectl ->>>>>>> Stashed changes COPY --from=builder /venv /venv ENV PATH="/venv/bin:${PATH}"