Run e2e tests with test using file names instead of marks

Makefile

@@ -62,6 +62,9 @@ precommit:
 switch:
 	@ scripts/dev/switch_context.sh $(context) $(additional_override)
 
+switcht:
+	@ scripts/dev/switch_context_by_test.sh $(test)
+
 # builds the Operator binary file and docker image and pushes it to the remote registry if using a remote registry. Deploys it to
 # k8s cluster
 operator: configure-operator build-and-push-operator-image

config/manifests/bases/mongodb-kubernetes.clusterserviceversion.yaml

@@ -8,9 +8,9 @@ metadata:
     certified: "true"
     containerImage: quay.io/mongodb/mongodb-kubernetes:1.1.0
     createdAt: ""
-    description: The MongoDB Controllers for Kubernetes enable easy deploys of 
-      MongoDB into Kubernetes clusters, using our management, monitoring and 
-      backup platforms, Ops Manager and Cloud Manager.
+    description: The MongoDB Controllers for Kubernetes enable easy deploys of MongoDB
+      into Kubernetes clusters, using our management, monitoring and backup platforms,
+      Ops Manager and Cloud Manager.
     features.operators.openshift.io/disconnected: "true"
     features.operators.openshift.io/fips-compliant: "false"
     features.operators.openshift.io/proxy-aware: "false"
@@ -51,8 +51,7 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ClusterConfiguration
-      - description: In a Replica Set deployment type, specifies the amount of 
-          members.
+      - description: In a Replica Set deployment type, specifies the amount of members.
         displayName: Members of a Replica Set
         path: members
         x-descriptors:
@@ -66,8 +65,7 @@ spec:
       - description: Project configuration for this deployment
         displayName: Ops Manager project configuration
         path: opsManager
-      - description: Name of the ConfigMap with the configuration for this 
-          project
+      - description: Name of the ConfigMap with the configuration for this project
         displayName: Ops Manager Project Configuration
         path: opsManager.configMapRef.name
         x-descriptors:
@@ -166,8 +164,7 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:text
         - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ClusterConfiguration
-      - description: In a Replica Set deployment type, specifies the amount of 
-          members.
+      - description: In a Replica Set deployment type, specifies the amount of members.
         displayName: Members of a Replica Set
         path: members
         x-descriptors:
@@ -181,8 +178,7 @@ spec:
       - description: Project configuration for this deployment
         displayName: Ops Manager project configuration
         path: opsManager
-      - description: Name of the ConfigMap with the configuration for this 
-          project
+      - description: Name of the ConfigMap with the configuration for this project
         displayName: Ops Manager Project Configuration
         path: opsManager.configMapRef.name
         x-descriptors:
@@ -194,8 +190,8 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
         - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ClusterConfiguration
-      - description: Optional. Specify whether to duplicate service objects 
-          among different Kubernetes clusters.
+      - description: Optional. Specify whether to duplicate service objects among
+          different Kubernetes clusters.
         displayName: Duplicate Service Objects
         path: duplicateServiceObjects
         x-descriptors:
@@ -256,8 +252,7 @@ spec:
         path: passwordSecretKeyRef.name
         x-descriptors:
         - urn:alm:descriptor:io.kubernetes:Secret
-      - displayName: Name of the MongoDB resource to which this user is 
-          associated.
+      - displayName: Name of the MongoDB resource to which this user is associated.
         path: mongodbResourceRef.name
         x-descriptors:
         - urn:alm:descriptor:io.kubernetes:mongodb
@@ -313,8 +308,8 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:io.kubernetes:Secret
         - urn:alm:descriptor:com.tectonic.ui:fieldGroup:OpsManagerConfiguration
-      - displayName: Secret to enable TLS for Ops Manager allowing it to serve 
-          traffic over HTTPS.
+      - displayName: Secret to enable TLS for Ops Manager allowing it to serve traffic
+          over HTTPS.
         path: security.tls.secretRef.name
         x-descriptors:
         - urn:alm:descriptor:io.kubernetes:Secret
@@ -324,8 +319,8 @@ spec:
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:number
         - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ApplicationDatabase
-      - displayName: Secret containing the TLS certificate signed by known or 
-          custom CA.
+      - displayName: Secret containing the TLS certificate signed by known or custom
+          CA.
         path: applicationDatabase.security.tls.secretRef.name
         x-descriptors:
         - urn:alm:descriptor:io.kubernetes:Secret

helm_chart/Chart.yaml

@@ -1,8 +1,7 @@
 apiVersion: v2
 name: mongodb-kubernetes
-description: MongoDB Controllers for Kubernetes translate the human knowledge of
-  creating a MongoDB instance into a scalable, repeatable, and standardized 
-  method.
+description: MongoDB Controllers for Kubernetes translate the human knowledge of creating
+  a MongoDB instance into a scalable, repeatable, and standardized method.
 version: 1.1.0
 kubeVersion: '>=1.16-0'
 type: application

scripts/__init__.py

@@ -0,0 +1 @@
+# Makes 'scripts' a Python package.

scripts/dev/contexts/root-context

@@ -116,3 +116,8 @@ export MDB_SEARCH_COMMUNITY_VERSION
 
 export MDB_SEARCH_COMMUNITY_NAME="mongodb-search-community"
 export MDB_SEARCH_COMMUNITY_REPO_URL="quay.io/mongodb"
+
+
+if [[ ${MDB_BASH_DEBUG:-0} == 1 ]]; then
+  export PS4='+(${BASH_SOURCE}:${LINENO})[^$?]: ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
+fi

scripts/dev/launch_e2e.sh

@@ -2,6 +2,7 @@
 
 set -Eeou pipefail
 
+test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x
 
 # The script launches e2e test. Note, that the Operator and necessary resources are deployed
 # inside the test
@@ -49,5 +50,3 @@ else
 fi
 
 title "E2e test ${test} is finished"
-
-

scripts/dev/set_env_context.sh

@@ -1,13 +1,14 @@
 #!/usr/bin/env bash
 
 set -Eeou pipefail
+test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x
 
 # shellcheck disable=1091
 source scripts/funcs/errors
 
 script_name=$(readlink -f "${BASH_SOURCE[0]}")
 script_dir=$(dirname "${script_name}")
-context_file="${script_dir}/../../.generated/context.export.env"
+context_file="$(realpath "${script_dir}/../../.generated/context.export.env")"
 
 if [[ ! -f ${context_file} ]]; then
     fatal "File ${context_file} not found! Make sure to follow this guide to get started: https://wiki.corp.mongodb.com/display/MMS/Setting+up+local+development+and+E2E+testing#SettinguplocaldevelopmentandE2Etesting-GettingStartedGuide(VariantSwitching)"

scripts/dev/switch_context_by_test.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+set -Eeou pipefail
+test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x
+
+usage() {
+  echo "Switch context by passing the test (evergreen task name or full evergreen task URL)."
+  echo "If there is more than one variant running given test, then fzf picker is used."
+  echo "Usage: $0 <test>"
+  echo "  <test> is a task name from .evergreen.yml (e.g. 'e2e_search_community_basic') or a full Evergreen task URL."
+}
+
+source scripts/funcs/errors
+
+list_pytest_marks() {
+  rg -g '*.py' -o --no-line-number --no-heading --replace '$1' -m 1 \
+    '@(?:pytest\.)?mark\.(e2e_[a-zA-Z0-9_]+)' \
+    docker/mongodb-kubernetes-tests
+}
+
+pick_test_by_file_mark_or_task_url() {
+    if ! test_list="$(list_pytest_marks | sort -u)"; then
+      echo "Couldn't list pytest marks."
+      echo "${test_list}"
+      return 1
+    fi
+
+    test=$(fzf --print-query --header-first --with-nth '{2}: {1}' -d ':' --accept-nth 2 \
+      --header "Select file or task to find contexts where its used. You can paste full task's evergreen url here" <<< "${test_list}") || true
+    if [[ -z ${test} ]]; then
+      echo "Aborted selecting test"
+      return 1
+    fi
+
+    # test may contain one or two lines (file:mark or just mark/url)
+    number_of_selected_lines=$(wc -l <<< "${test}")
+    if [[ ${number_of_selected_lines} -eq 2 ]]; then
+      test="$(tail -n 1 <<< "${test}")"
+    elif [[ ${number_of_selected_lines} -gt 2 ]]; then
+      echo "Too many lines selected: ${test}"
+      return 1
+    fi
+
+    echo "${test}"
+}
+
+main() {
+  test="${1:-}"
+
+  if [[ -z ${test} ]]; then
+    test=$(pick_test_by_file_mark_or_task_url)
+    echo "Selected test: ${test}"
+  fi
+
+  if [[ "${test}" = *spruce.mongodb.com* ]]; then
+    find_variant_arg="--task-url"
+  else
+    find_variant_arg="--task-name"
+  fi
+
+  if ! contexts=$(scripts/evergreen/run_python.sh scripts/python/find_test_variants.py "${find_variant_arg}" "${test}"); then
+    echo "Couldn't find any test contexts running test: ${test}"
+    echo "${contexts}"
+    exit 1
+  fi
+
+  echo "Found contexts that are running test: ${test}"
+  echo "${contexts}"
+
+  selected_context="${contexts}"
+  if [[ $(wc -l <<< "${contexts}") -gt 1 ]]; then
+    if ! selected_context=$(fzf --header "${test} runs in multiple variants/contexts. Select one to switch context into." --header-first --layout=reverse <<< "${contexts}"); then
+      echo "Aborted selecting context"
+      exit 1
+    fi
+  fi
+
+  scripts/dev/switch_context.sh "${selected_context}"
+}
+
+main "$@"

scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml

@@ -192,11 +192,19 @@ spec:
     image: {{ .Values.repo }}/mongodb-kubernetes-tests:{{ .Values.tag }}
     # Options to pytest command should go in the pytest.ini file.
     command: ["pytest"]
-    {{ if .Values.otel_endpoint }}
-    args: ["-vv", "-m", "{{ .Values.taskName }}", "--trace-parent", "00-{{ .Values.otel_trace_id }}-{{ .Values.otel_parent_id }}-01", "--export-traces"]
-    {{ else }}
-    args: ["-vv", "-m", "{{ .Values.taskName }}"]
-    {{ end }}
+    args:
+      - "-vv"
+      {{- if .Values.testFile }}
+      - "{{ .Values.testFile }}"
+      {{- else }}
+      - "-m"
+      - "{{ .Values.taskName }}"
+      {{- end }}
+      {{- if .Values.otel_endpoint }}
+      - "--trace-parent"
+      - "00-{{ .Values.otel_trace_id }}-{{ .Values.otel_parent_id }}-01"
+      - "--export-traces"
+      {{- end }}
     imagePullPolicy: Always
     volumeMounts:
       - name: results

scripts/evergreen/deployments/test-app/values.yaml

@@ -8,6 +8,10 @@ apiKey: omApiKey
 orgId: ""
 projectId: omProjectId
 tag:
+
+# if testFile is specified, then test is executed as pytest <testFile>
+testFile:
+# if testFile is empty, executes the test by fixture mark: pytest -m <taskName>
 taskName: ${TASK_NAME}
 
 pytest:

scripts/evergreen/e2e/single_e2e.sh

@@ -2,6 +2,7 @@
 
 set -Eeou pipefail
 
+test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x
 ##
 ## The script deploys a single test application and waits until it finishes.
 ## All the Operator deployment, configuration and teardown work is done in 'e2e' script
@@ -15,6 +16,23 @@ source scripts/funcs/operator_deployment
 
 check_env_var "TEST_NAME" "The 'TEST_NAME' must be specified to run the Operator single e2e test"
 
+find_test_file_by_fixture_mark() {
+  fixture_mark="$1"
+
+  cd docker/mongodb-kubernetes-tests
+  if ! test_files="$(grep -l -R "mark.${fixture_mark}$" --include '*.py')"; then
+    >&2 echo "Cannot find any test file containing a pytest fixture mark: ${fixture_mark}"
+    return 1
+  fi
+  number_of_files_matched=$(wc -l <<< "${test_files}")
+  if [[ ${number_of_files_matched} -gt 1 ]]; then
+    >&2 echo "Found more than one file with the same pytest fixture mark ${fixture_mark}:"
+    grep --color=auto --line-number --recursive -C2 "mark.${fixture_mark}$" --include '*.py' .
+    return 1
+  fi
+
+  echo -n "${test_files}"
+}
 
 deploy_test_app() {
     printenv
@@ -33,12 +51,17 @@ deploy_test_app() {
     BUILD_ID="${BUILD_ID:-default_build_id}"
     BUILD_VARIANT="${BUILD_VARIANT:-default_build_variant}"
 
+    if ! test_file=$(find_test_file_by_fixture_mark "${TASK_NAME}"); then
+      return 1
+    fi
+
     # note, that the 4 last parameters are used only for Mongodb resource testing - not for Ops Manager
     helm_params=(
         "--set" "taskId=${task_id:-'not-specified'}"
         "--set" "repo=${BASE_REPO_URL:=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev}"
         "--set" "namespace=${NAMESPACE}"
         "--set" "taskName=${task_name}"
+        "--set" "testFile=${test_file}"
         "--set" "tag=${tag}"
         "--set" "aws.accessKey=${AWS_ACCESS_KEY_ID}"
         "--set" "aws.secretAccessKey=${AWS_SECRET_ACCESS_KEY}"
@@ -128,7 +151,9 @@ deploy_test_app() {
 
     helm_params+=("--set" "opsManagerVersion=${ops_manager_version}")
 
-    helm template "scripts/evergreen/deployments/test-app" "${helm_params[@]}" > "${helm_template_file}" || exit 1
+    echo "Executing helm template:"
+    echo "helm template scripts/evergreen/deployments/test-app ${helm_params[*]}"
+    helm template "scripts/evergreen/deployments/test-app" "${helm_params[@]}" > "${helm_template_file}" || return 1
 
     cat "${helm_template_file}"
 
@@ -189,7 +214,9 @@ run_tests() {
 
     prepare_operator_config_map "${operator_context}"
 
-    deploy_test_app "${test_pod_context}"
+    if ! deploy_test_app "${test_pod_context}"; then
+      return 1
+    fi
 
     wait_until_pod_is_running_or_failed_or_succeeded "${test_pod_context}"
 

scripts/evergreen/flakiness-report.py → scripts/evergreen/flakiness_report.py

@@ -1,9 +1,9 @@
-import json
-import os
 import sys
 
 import requests
 
+from scripts.python.evergreen_api import get_evergreen_auth_headers
+
 EVERGREEN_API = "https://evergreen.mongodb.com/api"
 
 
@@ -18,16 +18,13 @@ def print_usage():
 
 
 def get_variants_with_retried_tasks() -> dict[str, list[dict]]:
-    evg_user = os.environ.get("EVERGREEN_USER", "")
-    api_key = os.environ.get("API_KEY", "")
-
-    if len(sys.argv) != 2 or evg_user == "" or api_key == "":
+    if len(sys.argv) != 2:
         print_usage()
-        exit(1)
+        raise RuntimeError("Exactly one argument (patch version number) must be provided")
 
     version = sys.argv[1]
+    headers = get_evergreen_auth_headers()
 
-    headers = {"Api-User": evg_user, "Api-Key": api_key}
     print("Fetching build variants...", file=sys.stderr)
     build_ids = requests.get(url=f"{EVERGREEN_API}/rest/v2/versions/{version}", headers=headers).json()
     build_statuses = [build_status for build_status in build_ids["build_variants_status"]]

scripts/funcs/checks

@@ -1,11 +1,5 @@
 #!/usr/bin/env bash
 
-pushd "${PWD}" > /dev/null || return
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "${DIR}" || return
-source errors
-popd > /dev/null || return
-
 check_env_var() {
     local var_name="$1"
     local msg="$2"